You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@beam.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2020/04/15 15:28:55 UTC
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1194
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1194/display/redirect?page=changes>
Changes:
[lcwik] [BEAM-2939] Update unbounded source as SDF wrapper to resume
[lcwik] fixup! Fix spotbugs warning
------------------------------------------
[...truncated 64.26 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, chardet, certifi, urllib3, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1586964530.130820981","description":"Error received from peer ipv4:127.0.0.1:40813","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0415152850-2a08ff7f_1473c7b1-c115-40ab-b025-46603527f1c0 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 19s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/6ogl26pxvep4s
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1444
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1444/display/redirect?page=changes>
Changes:
[zyichi] [BEAM-9603] Add timer family support to FnApiDoFnRunner
[zyichi] Address comments and add tests
[zyichi] comments
[aramaswamis] Fix typo copyLicenseScrips -> copyLicenseScripts
[github] [BEAM-7370] Upgrade sphinx to 3.0.3 (#11798)
------------------------------------------
[...truncated 64.41 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, certifi, urllib3, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0527000929-16d7041c_57bffa57-c4d3-441d-b5c4-884f2cd683d1 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 10s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/v6l4ujjezqgi6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1443
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1443/display/redirect?page=changes>
Changes:
[bhulette] add def args
[github] Fix typo Portabillity -> Portability
[github] [BEAM-10065] Fix beam release guide template (#11797)
[github] Fix typo in doc "and and" -> "and" (#11810)
[github] [BEAM-2939, BEAM-10057] Ensure that we can process an
------------------------------------------
[...truncated 65.38 KB...]
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Collecting crcmod<2.0,>=1.7
Using cached crcmod-1.7.tar.gz (89 kB)
Collecting dill<0.3.2,>=0.3.1.1
Using cached dill-0.3.1.1.tar.gz (151 kB)
Collecting fastavro<0.24,>=0.21.4
Downloading fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Collecting future<1.0.0,>=0.18.2
Using cached future-0.18.2.tar.gz (829 kB)
Collecting grpcio<2,>=1.12.1
Downloading grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Collecting hdfs<3.0.0,>=2.1.0
Using cached hdfs-2.5.8.tar.gz (41 kB)
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Downloading numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Downloading pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Collecting oauth2client<4,>=2.0.1
Using cached oauth2client-3.0.0.tar.gz (77 kB)
Collecting protobuf<4,>=3.5.0.post1
Downloading protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Collecting avro-python3!=1.9.2,<1.10.0,>=1.8.1
Using cached avro-python3-1.9.2.1.tar.gz (37 kB)
Collecting pyarrow<0.18.0,>=0.15.1
Downloading pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Building wheels for collected packages: crcmod, dill, future, hdfs, oauth2client, avro-python3
Building wheel for crcmod (setup.py): started
Building wheel for crcmod (setup.py): finished with status 'done'
Created wheel for crcmod: filename=crcmod-1.7-cp37-cp37m-linux_x86_64.whl size=30512 sha256=f691a10619480c294f40e5ebc724fe699886317fada1d885e2669fa64cd385a9
Stored in directory: /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f
Building wheel for dill (setup.py): started
Building wheel for dill (setup.py): finished with status 'done'
Created wheel for dill: filename=dill-0.3.1.1-py3-none-any.whl size=78530 sha256=1231f38399c1db3f3f3a7b662e5ef9e974d0ecf1a26c39520f21de0510d06cc3
Stored in directory: /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f
Building wheel for future (setup.py): started
Building wheel for future (setup.py): finished with status 'done'
Created wheel for future: filename=future-0.18.2-py3-none-any.whl size=491058 sha256=4ccc909043d03a6b250f6d5b2d0ae06fe3e9da7c345e6aa6446d7bc9c5b673c6
Stored in directory: /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0
Building wheel for hdfs (setup.py): started
Building wheel for hdfs (setup.py): finished with status 'done'
Created wheel for hdfs: filename=hdfs-2.5.8-py3-none-any.whl size=33213 sha256=aa68a2a557cc7df61992b9a6033df3fd333dccd586222f34f866eeafb09c1209
Stored in directory: /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859
Building wheel for oauth2client (setup.py): started
Building wheel for oauth2client (setup.py): finished with status 'done'
Created wheel for oauth2client: filename=oauth2client-3.0.0-py3-none-any.whl size=106383 sha256=aa35688ffb9bf67e704a87f33f6c1fda40872f6cf03b7c01838c2998b3bc1632
Stored in directory: /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1
Building wheel for avro-python3 (setup.py): started
Building wheel for avro-python3 (setup.py): finished with status 'done'
Created wheel for avro-python3: filename=avro_python3-1.9.2.1-py3-none-any.whl size=43513 sha256=1c1bc0af087ed7f820596949f6af7e0109291396f753a4ea7a980ac7b73b1e91
Stored in directory: /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1
Successfully built crcmod dill future hdfs oauth2client avro-python3
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, urllib3, idna, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:root:Waiting for grpc channel to be ready at localhost:52829.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0526202005-5bed06a0_7029c253-52fc-436a-a520-36a6ef7d279d failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 18s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/3utsnoxcpq26s
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1442
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1442/display/redirect?page=changes>
Changes:
[github] [BEAM-10073] Add pubsub performance tests results to the grafana
------------------------------------------
[...truncated 64.21 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, certifi, chardet, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0526121010-31070c62_36cb6ce4-9a11-481a-9b22-fe5e51789bd6 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 49s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ton26ouromlpu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1441
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1441/display/redirect>
Changes:
------------------------------------------
[...truncated 63.97 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, idna, urllib3, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0526060913-846e053b_cbb20ec3-5c4a-4d1e-8629-bdc5d528f10e failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 55s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/gq6k6fcqrlnvw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1440
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1440/display/redirect>
Changes:
------------------------------------------
[...truncated 64.43 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, chardet, urllib3, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0526000927-2277d7d2_f0a6a83b-444e-437f-b5f7-48184dad906f failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 5s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/yqhq34rjza4bw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1439
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1439/display/redirect>
Changes:
------------------------------------------
[...truncated 64.43 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/50/24/4d/4580ca4a299f1ad6fd63443e6e584cb21e9a07988e4aa8daac/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/59/b1/91/f02e76c732915c4015ab4010f3015469866c1eb9b14058d8e7/dill-0.3.1.1-cp37-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/8b/99/a0/81daf51dcd359a9377b110a8a886b3895921802d2fc1b2397e/future-0.18.2-cp37-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/fe/a7/05/23e3699975fc20f8a30e00ac1e515ab8c61168e982abe4ce70/hdfs-2.5.8-cp37-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/48/f7/87/b932f09c6335dbcf45d916937105a372ab14f353a9ca431d7d/oauth2client-3.0.0-cp37-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/5a/d3/be/86620c9dd3fca68986c33b9c616510289fc0abb81ec9aa70bd/avro_python3-1.9.2.1-cp37-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, urllib3, chardet, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0525180840-17d2944f_21480888-09b4-4097-a620-ecb9d0da01cf failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 21s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/q4mhnngzuynp6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1438
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1438/display/redirect?page=changes>
Changes:
[kamil.wasilewski] [BEAM-10058] Provide less strict assertion to make the test more
[github] [BEAM-10050] Change labels checked in VideoIntelligenceIT (#11775)
[github] [BEAM-9930] Update blog post for new Beam Summit Digital dates (#11772)
------------------------------------------
[...truncated 64.20 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, urllib3, chardet, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0525120919-9031241b_d83e2c4e-78be-4812-956a-230e5eebff00 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 56s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/od2ywlsbg52da
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1437
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1437/display/redirect>
Changes:
------------------------------------------
[...truncated 64.18 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, certifi, chardet, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0525060913-3678cef8_db2af6e5-4bc0-4e50-aef8-528c3d7845f5 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 53s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/gg7h4im4kri2w
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1436
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1436/display/redirect>
Changes:
------------------------------------------
[...truncated 64.41 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, urllib3, chardet, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0525000924-9c4804f1_f95e6a0d-99d2-4718-9e69-01e142d12f3a failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 3s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/7wpttauhezhhc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1435
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1435/display/redirect>
Changes:
------------------------------------------
[...truncated 64.18 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, urllib3, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0524180924-18090735_26c9bee3-eea9-49f8-82a6-312c4219cd41 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 4s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/wb45ewamfq7n6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1434
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1434/display/redirect>
Changes:
------------------------------------------
[...truncated 64.85 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, chardet, urllib3, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0524120907-cc2d2db8_956e6568-58aa-4e4e-83ce-485b7d609879 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 50s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/vtoeecxhnwq3w
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1433
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1433/display/redirect>
Changes:
------------------------------------------
[...truncated 64.42 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, urllib3, idna, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0524060907-87458b24_bef83f7d-aa59-410b-8a81-60e2a5ff3b90 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 47s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/kxvtowp7v6zju
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1432
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1432/display/redirect>
Changes:
------------------------------------------
[...truncated 64.41 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, chardet, certifi, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0524000916-7ce201e9_e67f9919-0ba6-4b70-b746-a6aa337299ce failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 59s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/w55kiyyyvpb7u
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1431
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1431/display/redirect>
Changes:
------------------------------------------
[...truncated 64.18 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, idna, certifi, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0523180937-1557f19a_ce7c101f-2fb7-49d0-b503-bbf402f2035a failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 18s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/wtb5a5uuukqo2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1430
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1430/display/redirect>
Changes:
------------------------------------------
[...truncated 64.18 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0523120922-5adce420_051109f1-1dc9-4409-9bfd-efab123fc454 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 3s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/tjnox5kkz6m6i
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1429
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1429/display/redirect>
Changes:
------------------------------------------
[...truncated 64.18 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, urllib3, certifi, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0523060913-9a84d672_130f47ba-a915-44b7-b336-8471323397bf failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 53s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/2hswdjt7ifspq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1428
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1428/display/redirect>
Changes:
------------------------------------------
[...truncated 64.18 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, idna, urllib3, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0523001037-bb6e51d5_68316e34-1528-4407-90ee-41850aae3b4f failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 18s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/jgjqw33m2wqr2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1427
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1427/display/redirect?page=changes>
Changes:
[github] [BEAM-10051] Move closed reader check after sentinel. (#11768)
[github] [BEAM-10056] Fix validation for struct CoGBKs (#11782)
------------------------------------------
[...truncated 64.41 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, idna, chardet, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0522180936-d43e0269_7286101d-361d-4b7c-8b53-a5481ce10547 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 16s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/5eihld5c3dnt4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1426
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1426/display/redirect>
Changes:
------------------------------------------
[...truncated 64.69 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, chardet, certifi, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0522120921-46391206_e60eff6a-2449-47c3-9366-51f7adc33a49 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
>>> FAILURE
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 2s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/k4vseexoeu27e
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1425
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1425/display/redirect?page=changes>
Changes:
[daniel.o.programmer] [BEAM-9978] Adding functionality and tests to Go offset range tracker.
------------------------------------------
[...truncated 64.46 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, certifi, chardet, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0522060943-c80d1968_53b237b1-3f54-4cb8-9ac7-e10b2eb2f7f7 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 22s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/d7txuc3gybnfi
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1424
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1424/display/redirect?page=changes>
Changes:
[kcweaver] [BEAM-10048] Clean up release guide.
[kcweaver] Remove checklist from build_release_candidate.sh.
[github] Include additional instructions for staging repository.
[heejong] [BEAM-10052] check hash and avoid duplicated artifacts
[github] Clarify the usage of sign_hash_python_wheels.sh.
[github] Website build: Remove -u root (#11625)
[ankurgoenka] [BEAM-10060] Upgrading Python containers to beam-master-20200521
[github] [BEAM-9722] added SnowflakeIO with Read operation (#11360)
------------------------------------------
[...truncated 64.23 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, idna, urllib3, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0522002415-9489e49e_a4ce46f6-8bfc-4553-8855-4ccb7616f38c failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 56s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/w5zbuacwgsi5q
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1423
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1423/display/redirect?page=changes>
Changes:
[rohde.samuel] Make GroupByKey a primitive
[rohde.samuel] Move ReifyWindows etc to DirectRunner, don't subclass from GBK
[rohde.samuel] remove references to _GroupByKeyOnly
[rohde.samuel] add back the ReifyWindows to the GBK, some internal tests depend on it.
[chamikara] Clarifies Dataflow execution environment model
[chamikara] Fixes typos.
[rohde.samuel] Skip checking input determinism for GBKs embedded in x-lang transforms.
[chamikara] Clarifies text.
[rehmanmuradali0] Update changes.md
------------------------------------------
[...truncated 63.95 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, certifi, chardet, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0521182335-656fc0dc_662bfffd-9e65-4c49-8cca-c70f724ea07d failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 16s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/tph2tlnqo5tce
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1422
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1422/display/redirect?page=changes>
Changes:
[github] Fix grammar / spelling
------------------------------------------
[...truncated 64.88 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, chardet, certifi, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0521120940-8c91da99_6d237f54-19e1-4a97-b845-67a8242ce50b failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 19s
67 actionable tasks: 52 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/rvcp43vqbnrwc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1421
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1421/display/redirect?page=changes>
Changes:
[veblush] Upgrades gcsio to 2.1.3
[veblush] Add grpc-alts and grpc-netty-shaded explicitly
[github] Merge pull request #11339 from [BEAM-9468] Fhir io
[github] [BEAM-9822] Simplify pipeline when batching is disabled. (#11529)
[bhulette] Moving to 2.23.0-SNAPSHOT on master branch.
[github] [BEAM-10028] Add support for the state backed iterable coder to the Java
------------------------------------------
[...truncated 66.92 KB...]
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.1-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, chardet, idna, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.1 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:root:Waiting for grpc channel to be ready at localhost:59243.
WARNING:root:Waiting for grpc channel to be ready at localhost:59243.
WARNING:root:Waiting for grpc channel to be ready at localhost:59243.
WARNING:root:Waiting for grpc channel to be ready at localhost:59243.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0521061312-6cd5d396_3fc053a8-424c-4091-a650-0c1cfcdae9be failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
>>> FAILURE
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 12m 56s
67 actionable tasks: 65 executed, 1 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/6y3f35dfglbb6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1420
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1420/display/redirect?page=changes>
Changes:
[github] Updated Videos and Podcasts page
[kcweaver] [BEAM-10038] Add script to mass-comment Jenkins triggers on PR
[kcweaver] Add mass comment script to release guide.
[aramaswamis] Remove curly quotes from documentation code samples
[kcweaver] Update test list.
[kcweaver] Update verify_release_build.sh to use mass_comment.py.
[kcweaver] Run yapf on mass_comment.py.
[github] Remove irrelevant instruction.
[github] Merge pull request #11596 from [BEAM-9856] Optimization/hl7v2 io list
[github] [BEAM-9679] Add Core Transforms section / GroupByKey lesson to the Go
[github] [BEAM-9699] Add test verifying we can use ZetaSQL in Python SqlTransform
[github] [BEAM-10037] BeamSqlExample.java fails to build (#11754)
[boyuanz] Implement GrowableOffsetRangeTracker
[github] [BEAM-9899] Fix some issues around storing schema `id` on user types
------------------------------------------
[...truncated 64.17 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, chardet, idna, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0521001347-53000207_f9440cf8-ac7e-4c83-a629-349829191b77 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 52s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/2u7dr4on3whow
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1419
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1419/display/redirect>
Changes:
------------------------------------------
[...truncated 64.19 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, urllib3, chardet, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0520181425-a1eb6377_6543b65f-01d6-4e55-a9e1-b3d4d288fc19 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 45s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/5ncac7ppnwxle
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1418
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1418/display/redirect?page=changes>
Changes:
[mxm] [BEAM-9900] Fix polling behavior in UnboundedSourceWrapper
[piotr.szuberski] [BEAM-9633] Add namespace parameter to MetricReader
[piotr.szuberski] [BEAM-9633] Add more parameters to pubsub matcher
[piotr.szuberski] [BEAM-9633] Add pubsubio performance tests
[piotr.szuberski] [BEAM-9633] Add jenkins job for python pubsubio tests
[mxm] [BEAM-9900] Add test case for polling idle readers
------------------------------------------
[...truncated 64.43 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, certifi, urllib3, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0520120939-3f1961b8_9c839300-f135-4704-a610-41af6c10d8ca failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 18s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/tbaoad5nfhxts
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1417
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1417/display/redirect>
Changes:
------------------------------------------
[...truncated 64.17 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, idna, certifi, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0520060855-36ec59ac_08ffbd0a-1943-4a25-a916-86b07f7cdd16 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 35s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/67m35cuxhfdca
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1416
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1416/display/redirect?page=changes>
Changes:
[rarokni] [BEAM-9770] Add BigQueryIO deadletter pattern Decouple .java and .py
[chamikara] Prevent Dataflow from starting multiple containers for the same image
[nielm] Disable grouping by default when streaming.
[github] [BEAM-9958] Linkage Checker to use exclusion files as baseline (#11674)
[daniel.o.programmer] [BEAM-9951] Using the builder pattern for Go synthetic config frontend
[github] [BEAM-9984] Support BIT_OR aggregation function in Beam SQL (#11737)
[github] Katas - Convert task description from HTML to Markdown (#11736)
------------------------------------------
[...truncated 64.17 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, chardet, certifi, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0520000920-deac2b63_d8382c53-2bb3-4ecc-8954-8c5a7dba2c15 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 58s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/2jxk2niriwvx4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1415
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1415/display/redirect?page=changes>
Changes:
[sruthi.sreekumar] Updated Flink runner version in NEXMark documentation
[kcweaver] Upgrade Dataflow Python container versions.
[github] [BEAM-9383] Staging Dataflow artifacts from environment (#11039)
[github] [BEAM-1589] Added @OnWindowExpiration annotation. (#11350)
------------------------------------------
[...truncated 64.86 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.3-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, urllib3, chardet, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.3 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0519181348-9b9c8af4_025a74fa-9c1c-4abf-84c1-78b253f076f6 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 162
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 5s
67 actionable tasks: 52 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/6gmh6hnk6m23m
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1414
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1414/display/redirect?page=changes>
Changes:
[pawel.pasterz] [BEAM-2530] Sql precommit compile and run on java 11
------------------------------------------
[...truncated 64.42 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.3-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.3 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0519121010-b32bd1e7_cd7da181-89c0-4b03-90d9-173148ee03d5 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 49s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/5xkohnt6uejgk
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1413
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1413/display/redirect?page=changes>
Changes:
[nielm] Populate all SpannerIO batching parameters in display data.
[nielm] Fix capitalization, clarify descriptions
[nielm] fix capitalization, clarify description Grouped
[nielm] Refactor to extract single method for popuplating displayData
[github] [BEAM-9339] Ensure that Dataflow's pipeline proto also contains the
[github] [BEAM-9363] TUMBLE as TVF (#10946)
------------------------------------------
[...truncated 64.40 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.3-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, urllib3, chardet, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.3 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0519060942-1db6b5d1_ec4f372c-a930-4ca7-9961-5210313c5b82 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 22s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/jhoi4h7ftvqnw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1412
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1412/display/redirect?page=changes>
Changes:
[robinyqiu] Support ZetaSQL DATE type as a Beam LogicalType
[hsuryawirawan] Remove all answer placeholder checks as they can be confusing at times
[hsuryawirawan] Update course in Stepik
[ihr] [BEAM-10018] Fix timestamps in windowing kata
[ihr] [BEAM-10018] Kata failing due to failed parsing
[ihr] Remove unused import
[ihr] Add missing dependency
[ihr] Fix member variable name in Kata documentation
[ihr] Fix placeholder location
[github] [BEAM-9964] Update CHANGES.md (#11743)
[github] [BEAM-9577] Artifact v2 support for uber jars. (#11708)
------------------------------------------
[...truncated 64.42 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.3-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, chardet, certifi, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.3 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0519002355-524eed03_c24199d6-c097-4ef6-91f9-902810e5ec2c failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 56s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/wdq6welrm4e5c
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1411
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1411/display/redirect?page=changes>
Changes:
[mxm] [BEAM-6733] Add pipeline option to flush bundle data before
[github] Python3 fix - convert dict.keys() to list before indexing (#11733)
[github] Updates google-apitools and httplib2 (#11726)
------------------------------------------
[...truncated 63.94 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.3-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, chardet, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.3 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0518181009-494ba29d_815c744a-9071-4c44-a2f2-b34d06e784a7 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 46s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/6jv4qy5vjznem
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1410
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1410/display/redirect?page=changes>
Changes:
[github] [BEAM-2530] Implement Zeta SQL precommit compile tests and run on java
------------------------------------------
[...truncated 64.21 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, certifi, urllib3, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0518121023-2fede3e1_5dcf5d03-72ea-40cb-9c4d-569a730a8f72 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 38s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/diuad35hto74s
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1409
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1409/display/redirect>
Changes:
------------------------------------------
[...truncated 64.23 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, idna, chardet, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0518060906-59c02217_1154ce91-9fc1-47c2-9746-99696dc3a064 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 46s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ifb6qkqo2z6ik
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1408
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1408/display/redirect>
Changes:
------------------------------------------
[...truncated 64.45 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, certifi, idna, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0518000911-ea038ee1_fd18e221-cd2d-49cb-abea-a081641f7a67 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 51s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/gndlw6yahfrfy
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1407
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1407/display/redirect>
Changes:
------------------------------------------
[...truncated 64.22 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, urllib3, chardet, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0517181004-4135acbe_8fd1078c-2991-4a6d-a82c-481232951e9a failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 39s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/4gkobiyivsbaq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1406
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1406/display/redirect>
Changes:
------------------------------------------
[...truncated 64.22 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, chardet, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0517120908-618d9567_affd7342-a8b4-42b4-819d-c29f37bbe954 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 48s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/74tp3mf5l5rx6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1405
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1405/display/redirect?page=changes>
Changes:
[github] [BEAM-2939] Ensure that we update the watermark even when no elements
------------------------------------------
[...truncated 599.96 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.4.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, urllib3, certifi, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0517060945-68e77af8_02068a49-11f5-41fc-b37b-910c21d2aff4 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 26s
67 actionable tasks: 61 executed, 5 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ner5xeae3tn4m
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1404
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1404/display/redirect>
Changes:
------------------------------------------
[...truncated 64.44 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.3.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0517000858-90f8c5e7_6b932c55-bcd3-4fb2-933b-21bbdfb0b2bc failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 39s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/7ns6uegdmtqwc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1403
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1403/display/redirect>
Changes:
------------------------------------------
[...truncated 64.44 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.3.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, urllib3, idna, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0516181026-72c340c_48a07755-977c-4831-bd6c-4f2e640a36ba failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 5s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/cmzcgecfgz776
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1402
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1402/display/redirect?page=changes>
Changes:
[github] Merge pull request #11725: [BEAM-10015] Fix output timestamp on dataflow
------------------------------------------
[...truncated 65.10 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.3.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, urllib3, certifi, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0516121039-5c09e904_9e7366aa-ab5d-4720-a7f8-99139745bc11 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 18s
67 actionable tasks: 52 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/bg4di42nl3aca
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1401
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1401/display/redirect?page=changes>
Changes:
[daniel.o.programmer] [BEAM-9951] Creating a synthetic step for the Go SDK.
------------------------------------------
[...truncated 64.44 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.3.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, chardet, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0516060905-54f5fc92_03c824da-4cf8-4020-964b-a7fa8825fb00 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
>>> FAILURE
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 45s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/nsrnz4dudnsbk
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1400
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1400/display/redirect?page=changes>
Changes:
[rohde.samuel] Change TestStreamImpl to a producer/consumer pattern
[apilloud] [BEAM-9522] Reject FULL JOIN ON false
[omarismail] [BEAM-9964] Setting --workerCacheMB to make its way to the
[omarismail] [BEAM-9964] Added testMaxWeight method in WIndmillStateCacheTest as per
[jiangxuehua1213] [BEAM-9136] Update doc about docker image license pulling
[bntnam.app] Modified the codeblock background
[github] support isRelease tag for docker build command & update release guide
[github] [BEAM-1866] Enable validation of PAssert counts in Java validates runner
[daniel.o.programmer] [BEAM-9951] Creating a synthetic source for the Go SDK.
------------------------------------------
[...truncated 64.41 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Downloading protobuf-3.12.0-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.3.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, urllib3, idna, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.0 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0516000942-d3113f4_12068ece-c06e-4901-8dc8-db8728a3b007 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
>>> FAILURE
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 21s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/akthp35k5vmso
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1399
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1399/display/redirect?page=changes>
Changes:
[mxm] [BEAM-9966] Use dataproc staging bucket for checkpoints directory
[mxm] [BEAM-9966] Remove unnecessary Reshuffles
[kamil.wasilewski] Make Post-Commits status dashboard compatible with the latest Grafana
[mxm] [BEAM-9966] Propagate the keys to the load generator correctly
[github] [BEAM-2939] Ensure that we don't checkpoint the initial restriction as
------------------------------------------
[...truncated 64.40 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Downloading fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.3.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, certifi, chardet, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0515183152-1a79c9f9_483b0a93-b4fb-4ad8-b107-513e84761754 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 56s
67 actionable tasks: 52 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/hsn5kop5fitru
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1398
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1398/display/redirect?page=changes>
Changes:
[pawel.pasterz] [BEAM-2530] PreCommit Dataflow examples - test compiled with java 11
[github] Update release notes link for 2.20.0
------------------------------------------
[...truncated 64.45 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.3.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, chardet, certifi, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0515120926-285b35e5_ccf1086b-ba6d-4a64-8129-4d543de044ae failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 6s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ixlmr2dipiwzc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1397
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1397/display/redirect?page=changes>
Changes:
[yifanzou] Run Inventory job twice a day at 6 and 18.
------------------------------------------
[...truncated 64.86 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.3.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0515061013-64c9e1cf_db2a4c6f-bb5d-4cf7-af2e-a51b4101b55a failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 54s
67 actionable tasks: 52 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/qlqky7zpqnywg
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1396
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1396/display/redirect?page=changes>
Changes:
[pabloem] [BEAM-9967] Adding support for BQ labels on Query/Export jobs. (Roll
[github] [BEAM-9634] Add natural language analysis transform (#11611)
[github] [BEAM-9430] Fixes the bounds of initial watermark set to estimators
[kcweaver] [BEAM-9993] Add option defaults for Flink Python tests.
[github] [BEAM-9941] Added a test of a GBK followed by a Flatten with an unknown
[github] Clarify pubsub IO comment about timestamps (#11672)
[iemejia] [BEAM-9833] Add yamllint config
[iemejia] [BEAM-9833] Fix .asf.yaml issues, sort labels and disable rebase button
[github] Clarifies an error message in Katas to explain what is actually wrong.
[github] Merge pull request #11210 from [BEAM-8949] SpannerIO integration tests
[github] [BEAM-9876] Migrate the Beam website from Jekyll to Hugo to enable
[github] Update the range for pyarrow to qualify pyarrow 0.17.x (#11699)
------------------------------------------
[...truncated 64.57 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.0-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.3.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, idna, urllib3, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.17.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0515001052-a9a781bf_4f5bd04e-d6a2-40b7-9011-9e9f5164c47d failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 54s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/dzokuwnsen3ws
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1395
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1395/display/redirect>
Changes:
------------------------------------------
[...truncated 63.98 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.3.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, idna, urllib3, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0514181655-6f7f7a76_19a81947-e610-4e7e-9315-e08c0c347626 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
>>> FAILURE
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 8s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/gut2m6tkfja5w
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1394
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1394/display/redirect>
Changes:
------------------------------------------
[...truncated 64.43 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.3.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, chardet, certifi, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0514120916-b894cde3_8ebcea62-1cf5-41ce-87ee-ca315d190c74 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 54s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/46stvwtlfnzea
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1393
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1393/display/redirect?page=changes>
Changes:
[github] [BEAM-9928] Add qualified import name to task descriptors (#11690)
------------------------------------------
[...truncated 64.20 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.3.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, chardet, certifi, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0514060941-3c7606f6_a938335c-eac3-4e3a-9e40-ef48565d83fa failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 20s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/7ftgtljwljdg6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1392
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1392/display/redirect?page=changes>
Changes:
[sniemitz] Allow users of AvroIO to specify a custom DatumReader implementation
[tysonjh] Disambiguate method filter for toArray.
[pabloem] Revert "Merge pull request #11673 from [BEAM-9967] Adding support for BQ
[thw] Bump Flink 1.10 version
[kcweaver] [BEAM-9975] Log invalid dictionary after unsuccessful struct conversion.
[github] [BEAM-9945] Ensure that the read index represents the number of fully
[github] [BEAM-9942] reduce groovy code duplication in python test-suites
[boyuanz] [BEAM-9935] Respect allowed split points in Java
[github] Switch to %s
[github] [BEAM-9001, BEAM-6327] Ensure that all transforms (except for required
------------------------------------------
[...truncated 65.97 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.3.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, urllib3, certifi, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0514000934-1d0ed862_e2fd46b4-16da-4a1f-8566-d9101b4699fd failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 14s
67 actionable tasks: 58 executed, 8 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/illxwzn2mpj7o
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1391
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1391/display/redirect?page=changes>
Changes:
[kawaigin] Use csv reader instead of split to read csv data.
------------------------------------------
[...truncated 64.38 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.3.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0513181231-67e32949_c474db1b-374d-43b4-8f2d-2f21f12c1911 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 25s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/axxt2ugvhiwd6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1390
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1390/display/redirect>
Changes:
------------------------------------------
[...truncated 63.94 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/50/24/4d/4580ca4a299f1ad6fd63443e6e584cb21e9a07988e4aa8daac/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/59/b1/91/f02e76c732915c4015ab4010f3015469866c1eb9b14058d8e7/dill-0.3.1.1-cp37-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/8b/99/a0/81daf51dcd359a9377b110a8a886b3895921802d2fc1b2397e/future-0.18.2-cp37-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/fe/a7/05/23e3699975fc20f8a30e00ac1e515ab8c61168e982abe4ce70/hdfs-2.5.8-cp37-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/48/f7/87/b932f09c6335dbcf45d916937105a372ab14f353a9ca431d7d/oauth2client-3.0.0-cp37-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/5a/d3/be/86620c9dd3fca68986c33b9c616510289fc0abb81ec9aa70bd/avro_python3-1.9.2.1-cp37-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.2.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, chardet, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0513121501-d92b1250_c2f69dca-3610-4feb-ac04-352b4902648f failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 46s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/b4l554ypdmhfe
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1389
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1389/display/redirect?page=changes>
Changes:
[github] [BEAM-9959] Root Transform fixes (#11686)
------------------------------------------
[...truncated 64.41 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/50/24/4d/4580ca4a299f1ad6fd63443e6e584cb21e9a07988e4aa8daac/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/59/b1/91/f02e76c732915c4015ab4010f3015469866c1eb9b14058d8e7/dill-0.3.1.1-cp37-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/8b/99/a0/81daf51dcd359a9377b110a8a886b3895921802d2fc1b2397e/future-0.18.2-cp37-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/fe/a7/05/23e3699975fc20f8a30e00ac1e515ab8c61168e982abe4ce70/hdfs-2.5.8-cp37-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/48/f7/87/b932f09c6335dbcf45d916937105a372ab14f353a9ca431d7d/oauth2client-3.0.0-cp37-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/5a/d3/be/86620c9dd3fca68986c33b9c616510289fc0abb81ec9aa70bd/avro_python3-1.9.2.1-cp37-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.2.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, chardet, urllib3, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0513061529-a41ca0c7_5abcfa5a-8911-4dd8-9060-162285a8733a failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 8s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/3g3d6c2rk3b66
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1388
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1388/display/redirect?page=changes>
Changes:
[mxm] [BEAM-9164] Re-enable UnboundedSourceWrapper#testWatermarkEmission test
[github] Merge pull request #11673 from [BEAM-9967] Adding support for BQ labels
[github] [BEAM-9622] Add Python SqlTransform test that joins tagged PCollections
------------------------------------------
[...truncated 64.88 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/50/24/4d/4580ca4a299f1ad6fd63443e6e584cb21e9a07988e4aa8daac/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/59/b1/91/f02e76c732915c4015ab4010f3015469866c1eb9b14058d8e7/dill-0.3.1.1-cp37-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/8b/99/a0/81daf51dcd359a9377b110a8a886b3895921802d2fc1b2397e/future-0.18.2-cp37-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/fe/a7/05/23e3699975fc20f8a30e00ac1e515ab8c61168e982abe4ce70/hdfs-2.5.8-cp37-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/48/f7/87/b932f09c6335dbcf45d916937105a372ab14f353a9ca431d7d/oauth2client-3.0.0-cp37-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/5a/d3/be/86620c9dd3fca68986c33b9c616510289fc0abb81ec9aa70bd/avro_python3-1.9.2.1-cp37-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.2.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, idna, urllib3, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0513000856-97ad5329_41645b27-78f6-4781-84c6-f3c8c49ae346 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 37s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/y2e5ubuf7p6r6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1387
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1387/display/redirect?page=changes>
Changes:
[github] Merge pull request #11567: [BEAM-8132] Report Python metrics to InfluxDB
[kamil.wasilewski] Fix InfluxDB measurement names to match those in Grafana dashboards
[github] [BEAM-9945] Report data channel progress via a designated counter.
[github] [BEAM-9577] Update Java Runners to handle dependency-based artifact
------------------------------------------
[...truncated 65.21 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.22.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.2.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0512181226-ba830381_3375aab1-9ba3-4b89-b44f-02af2017e1a1 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 46s
67 actionable tasks: 55 executed, 11 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/eoanudfx5faas
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1386
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1386/display/redirect?page=changes>
Changes:
[kamil.wasilewski] [BEAM-6710] Add landing page with links to relevant dashboards
[mxm] [BEAM-9930] Add image to Beam Summit blog post
[mxm] [BEAM-9947] Store updated ParDoPayload for length-prefixed timer coders
[mxm] [BEAM-9947] Add unit test to check length prefixing of timer key
[kamil.wasilewski] Fix metrics reporting in HDFS IO IT tests
[mxm] [BEAM-9963] Fix coder type in failing ParDo streaming load test
------------------------------------------
[...truncated 65.28 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.2.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, urllib3, chardet, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1589285367.908553557","description":"Error received from peer ipv4:127.0.0.1:36887","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0512120928-302970e3_d148d409-2fbc-44b9-b2fc-722a6cded484 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 8s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/4tqvdvcocpsxa
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1385
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1385/display/redirect?page=changes>
Changes:
[robertwb] Make grpc-requiring class definition optional.
[daniel.o.programmer] [BEAM-9642] Adding Go SDF fallback for unexpanded SDFs.
------------------------------------------
[...truncated 65.04 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.2.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, certifi, idna, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1589263766.554580616","description":"Error received from peer ipv4:127.0.0.1:41301","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0512060926-bf4bec0d_13830b15-0d0c-4861-b57f-0dd6a5300780 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 6s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/rqhmhi32bq36w
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1384
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1384/display/redirect?page=changes>
Changes:
[sjvanrossum] [BEAM-9776] Fixes filesystem not found error
[kcweaver] [BEAM-9875] Use FlinkRunner instead of PortableRunner in cross-language
[kcweaver] Remove runner check in xlang wordcount.
[kcweaver] [BEAM-9835] [Portable Spark] Broadcast a PCollection at most once.
[robertwb] [BEAM-9935] Respect allowed split points in Python.
[github] Add License file for the Go SDK for pkg.go.dev
[robertwb] More comments.
[rohde.samuel] Return empty iterator on empty cache
[borzoo.esmailloo] [BEAM-9216] Upgrade katas gradle to version 5.2.1
[github] [BEAM-9887] Throw IllegalArgumentException when building Row with
[klk] [BEAM-9952] Revert "remove PipelineRunner.fromOptions call"
[github] Merge pull request #11637 from Waiting for BQ Query and Export jobs for
[github] [BEAM-9884] Add options to SqlTransform for specifying planner (#11647)
------------------------------------------
[...truncated 65.23 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.2.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1589242224.057205785","description":"Error received from peer ipv4:127.0.0.1:44861","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0512001024-b952f08b_1441a4ca-d05d-45b1-9dae-cd38cfb7a698 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 3s
67 actionable tasks: 52 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/rzxi7fmq3bmmy
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1383
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1383/display/redirect?page=changes>
Changes:
[kamil.wasilewski] Fix InfluxDB credentials binding
------------------------------------------
[...truncated 65.23 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/50/24/4d/4580ca4a299f1ad6fd63443e6e584cb21e9a07988e4aa8daac/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/59/b1/91/f02e76c732915c4015ab4010f3015469866c1eb9b14058d8e7/dill-0.3.1.1-cp37-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/8b/99/a0/81daf51dcd359a9377b110a8a886b3895921802d2fc1b2397e/future-0.18.2-cp37-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/fe/a7/05/23e3699975fc20f8a30e00ac1e515ab8c61168e982abe4ce70/hdfs-2.5.8-cp37-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/48/f7/87/b932f09c6335dbcf45d916937105a372ab14f353a9ca431d7d/oauth2client-3.0.0-cp37-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/5a/d3/be/86620c9dd3fca68986c33b9c616510289fc0abb81ec9aa70bd/avro_python3-1.9.2.1-cp37-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.2.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, chardet, urllib3, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1589220545.287168462","description":"Error received from peer ipv4:127.0.0.1:35401","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0511180905-c32d8193_a9e12bcd-1f22-4b2c-b665-184638945fad failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 46s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/7y3xakb3t7wkg
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1382
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1382/display/redirect?page=changes>
Changes:
[sebastian.graca] [BEAM-9439] Return split instead of total backlog size
------------------------------------------
[...truncated 65.29 KB...]
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.2.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, certifi, urllib3, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:root:Waiting for grpc channel to be ready at localhost:47333.
WARNING:root:Waiting for grpc channel to be ready at localhost:47333.
WARNING:root:Waiting for grpc channel to be ready at localhost:47333.
WARNING:root:Waiting for grpc channel to be ready at localhost:47333.
WARNING:root:Waiting for grpc channel to be ready at localhost:47333.
WARNING:root:Waiting for grpc channel to be ready at localhost:47333.
WARNING:root:Waiting for grpc channel to be ready at localhost:47333.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1589199144.045591719","description":"Error received from peer ipv4:127.0.0.1:35923","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0511121225-1d715780_ab77800a-5d91-4ee6-839c-237ee25e948e failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
>>> FAILURE
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 12m 10s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/qqk63iwcvvycu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1381
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1381/display/redirect>
Changes:
------------------------------------------
[...truncated 65.05 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.2.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, chardet, idna, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1589177364.631536237","description":"Error received from peer ipv4:127.0.0.1:38119","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0511060924-c5d95c61_cdbce78e-6eda-4b5b-919c-ab8a1b7502ef failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
>>> FAILURE
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 6s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/y5khwassw26q6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1380
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1380/display/redirect>
Changes:
------------------------------------------
[...truncated 65.28 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.2.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, chardet, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1589155761.736960683","description":"Error received from peer ipv4:127.0.0.1:45397","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0511000921-ebb6b1b3_c04e41a2-231a-43cf-b8c2-6913d04cb478 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 2s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ti5dcllcrq7hc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1379
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1379/display/redirect>
Changes:
------------------------------------------
[...truncated 65.28 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, idna, urllib3, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1589134179.246206531","description":"Error received from peer ipv4:127.0.0.1:35627","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0510180939-ee663793_c595d105-7ade-4218-9425-7f209c6504a4 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 19s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/cop7vfb2qqgmk
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1378
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1378/display/redirect?page=changes>
Changes:
[mxm] [BEAM-9930] Beam Summit Digital 2020 announcement on blog
[mxm] Correct URL in blog post
[matthias.baetens] Minor (markup) changes
[mxm] [BEAM-9930] Correct authors of beam summit digital blog post
------------------------------------------
[...truncated 65.05 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, certifi, chardet, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1589112584.607593421","description":"Error received from peer ipv4:127.0.0.1:42343","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0510120944-f26974c6_d11a44d4-88f1-4482-9851-2f39ab467419 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 24s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/2oaapk36jyb7c
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1377
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1377/display/redirect>
Changes:
------------------------------------------
[...truncated 65.48 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, idna, certifi, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1589090957.027157464","description":"Error received from peer ipv4:127.0.0.1:36207","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0510060917-39fe2acb_32bfb4b1-0bc5-4ede-8ecc-07ec7208c041 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 56s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/njlorhw2aez6w
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1376
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1376/display/redirect>
Changes:
------------------------------------------
[...truncated 65.02 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, idna, urllib3, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1589069361.919911782","description":"Error received from peer ipv4:127.0.0.1:45905","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0510000922-c1f0651d_f46f19d0-16f8-4aad-b3cf-aa29753732ef failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 3s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/kngz543gmekm2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1375
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1375/display/redirect?page=changes>
Changes:
[chambers] Add a BeamJava test with Flatten with different input and output Coders
[jferriero] check lastAttemptedOffset non-null
------------------------------------------
[...truncated 64.53 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, idna, urllib3, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1589047842.337474120","description":"Error received from peer ipv4:127.0.0.1:37505","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0509181042-db103e50_058af4a6-cd74-495c-818c-19c910570928 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
>>> FAILURE
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 22s
67 actionable tasks: 52 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/nfsy57zzig3bw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1374
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1374/display/redirect>
Changes:
------------------------------------------
[...truncated 65.01 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, certifi, chardet, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1589026177.284770852","description":"Error received from peer ipv4:127.0.0.1:43425","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0509120937-95041da4_10343f3c-d527-4899-a641-cd092c55ec80 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 17s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/kvpgfby7fhwq6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1373
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1373/display/redirect?page=changes>
Changes:
[zyichi] [BEAM-9940] Set timer family spec for TimerDeclarations in dataflow
------------------------------------------
[...truncated 65.24 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, chardet, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1589004558.848644626","description":"Error received from peer ipv4:127.0.0.1:35635","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0509060919-99a45135_cd97a07f-389e-4d24-8190-8e5f3f4551de failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 58s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ifickl3jktxxa
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1372
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1372/display/redirect?page=changes>
Changes:
[kcweaver] [BEAM-4782] Remove workaround in Python multimap tests.
[github] [BEAM-9883] Refactor SDF test restrictions. (#11605)
[github] [BEAM-3288] Add suggested fix to error message (#11622)
------------------------------------------
[...truncated 65.25 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, idna, chardet, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588982980.293280474","description":"Error received from peer ipv4:127.0.0.1:41123","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0509000940-c430fed8_8dc152b4-4cb7-4a6a-bcc2-b2a90a990dd4 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 20s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/m66iphsl6unz6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1371
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1371/display/redirect?page=changes>
Changes:
[nielm] Remove spurious error message.
[alxavier] [BEAM-9911]Replace SpannerIO.write latency counter to distribution
[kamil.wasilewski] [BEAM-8132, BEAM-8133] Apply InfluxDB pipeline options in Load Tests and
[kamil.wasilewski] [BEAM-8132, BEAM-8133] Assume no trailing slash after a hostname
[iemejia] [BEAM-2530] Compile and run tests on java 11 for Precommit portability
------------------------------------------
[...truncated 34.49 KB...]
Collecting importlib-metadata<2,>=0.12; python_version < "3.8"
Using cached importlib_metadata-1.6.0-py2.py3-none-any.whl (30 kB)
Collecting enum34>=1.0.4; python_version < "3.4"
Using cached enum34-1.1.10-py2-none-any.whl (11 kB)
Collecting futures>=2.2.0; python_version < "3.2"
Using cached futures-3.3.0-py2-none-any.whl (16 kB)
Collecting singledispatch; python_version < "3.4"
Using cached singledispatch-3.4.0.3-py2.py3-none-any.whl (12 kB)
Collecting zipp>=0.4; python_version < "3.8"
Using cached zipp-1.2.0-py2.py3-none-any.whl (4.8 kB)
Collecting typing; python_version < "3.5"
Using cached typing-3.7.4.1-py2-none-any.whl (26 kB)
Processing /home/jenkins/.cache/pip/wheels/58/2c/26/52406f7d1f19bcc47a6fbd1037a5f293492f5cf1d58c539edb/scandir-1.10.0-cp27-cp27mu-linux_x86_64.whl
Collecting configparser>=3.5; python_version < "3"
Using cached configparser-4.0.2-py2.py3-none-any.whl (22 kB)
Installing collected packages: six, filelock, distlib, scandir, pathlib2, singledispatch, contextlib2, zipp, configparser, importlib-metadata, typing, importlib-resources, appdirs, virtualenv, pluggy, py, toml, tox, enum34, futures, grpcio, protobuf, grpcio-tools, future, mypy-protobuf
> Task :vendor:sdks-java-extensions-protobuf:compileJava FROM-CACHE
> Task :vendor:sdks-java-extensions-protobuf:classes UP-TO-DATE
> Task :sdks:java:extensions:google-cloud-platform-core:compileJava FROM-CACHE
> Task :sdks:java:extensions:google-cloud-platform-core:classes UP-TO-DATE
> Task :runners:core-construction-java:compileJava FROM-CACHE
> Task :runners:core-construction-java:classes UP-TO-DATE
> Task :sdks:java:extensions:google-cloud-platform-core:jar
> Task :sdks:java:fn-execution:compileJava FROM-CACHE
> Task :sdks:java:fn-execution:classes UP-TO-DATE
> Task :sdks:java:fn-execution:jar
> Task :vendor:sdks-java-extensions-protobuf:shadowJar
> Task :runners:core-construction-java:jar
> Task :sdks:go:resolveBuildDependencies
Resolving golang.org/x/crypto: commit='d9133f5469342136e669e85192a26056b587f503', urls=[https://go.googlesource.com/crypto]
> Task :runners:core-java:compileJava FROM-CACHE
> Task :runners:core-java:classes UP-TO-DATE
> Task :runners:core-java:jar
> Task :sdks:java:harness:compileJava FROM-CACHE
> Task :sdks:java:harness:classes UP-TO-DATE
> Task :sdks:java:harness:jar
> Task :sdks:python:setupVirtualenv
Successfully installed appdirs-1.4.3 configparser-4.0.2 contextlib2-0.6.0.post1 distlib-0.3.0 enum34-1.1.10 filelock-3.0.12 future-0.18.2 futures-3.3.0 grpcio-1.28.1 grpcio-tools-1.14.2 importlib-metadata-1.6.0 importlib-resources-1.5.0 mypy-protobuf-1.18 pathlib2-2.3.5 pluggy-0.13.1 protobuf-3.11.3 py-1.8.1 scandir-1.10.0 singledispatch-3.4.0.3 six-1.14.0 toml-0.10.0 tox-3.11.1 typing-3.7.4.1 virtualenv-20.0.20 zipp-1.2.0
> Task :sdks:go:resolveBuildDependencies
Resolving golang.org/x/debug: commit='95515998a8a4bd7448134b2cb5971dbeb12e0b77', urls=[https://go.googlesource.com/debug]
Resolving golang.org/x/net: commit='2fb46b16b8dda405028c50f7c7f0f9dd1fa6bfb1', urls=[https://go.googlesource.com/net]
Resolving golang.org/x/oauth2: commit='a032972e28060ca4f5644acffae3dfc268cc09db', urls=[https://go.googlesource.com/oauth2]
Resolving golang.org/x/sync: commit='fd80eb99c8f653c847d294a001bdf2a3a6f768f5', urls=[https://go.googlesource.com/sync]
Resolving golang.org/x/sys: commit='37707fdb30a5b38865cfb95e5aab41707daec7fd', urls=[https://go.googlesource.com/sys]
Resolving cached github.com/coreos/etcd: commit='11214aa33bf5a47d3d9d8dafe0f6b97237dfe921', urls=[https://github.com/coreos/etcd.git, git@github.com:coreos/etcd.git]
Resolving cached github.com/coreos/etcd: commit='11214aa33bf5a47d3d9d8dafe0f6b97237dfe921', urls=[https://github.com/coreos/etcd.git, git@github.com:coreos/etcd.git]
> Task :sdks:java:harness:shadowJar
> Task :sdks:python:sdist
setup.py:262: UserWarning: You are using Apache Beam with Python 2. New releases of Apache Beam will soon support Python 3 only.
'You are using Apache Beam with Python 2. '
<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/1922375555/local/lib/python2.7/site-packages/setuptools/dist.py>:476: UserWarning: Normalizing '2.22.0.dev' to '2.22.0.dev0'
normalized_version,
INFO:gen_protos:Regenerating Python proto definitions (no output files).
INFO:gen_protos:Found protoc_gen_mypy at <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/1922375555/bin/protoc-gen-mypy>
metrics.proto: warning: Import google/protobuf/timestamp.proto but not used.
beam_fn_api.proto: warning: Import google/protobuf/descriptor.proto but not used.
beam_fn_api.proto: warning: Import google/protobuf/wrappers.proto but not used.
beam_interactive_api.proto: warning: Import google/protobuf/timestamp.proto but not used.
Writing mypy to endpoints_pb2.pyi
Writing mypy to external_transforms_pb2.pyi
Writing mypy to beam_provision_api_pb2.pyi
Writing mypy to beam_runner_api_pb2.pyi
Writing mypy to standard_window_fns_pb2.pyi
Writing mypy to beam_artifact_api_pb2.pyi
Writing mypy to beam_fn_api_pb2.pyi
Writing mypy to metrics_pb2.pyi
Writing mypy to schema_pb2.pyi
Writing mypy to beam_job_api_pb2.pyi
Writing mypy to beam_interactive_api_pb2.pyi
Writing mypy to beam_expansion_api_pb2.pyi
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
RefactoringTool: Skipping optional fixer: idioms
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
RefactoringTool: Skipping optional fixer: ws_comma
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_interactive_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_runner_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_runner_api_pb2_grpc.py>
RefactoringTool: No changes to <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/endpoints_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/external_transforms_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/metrics_pb2.py>
RefactoringTool: No changes to <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/schema_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/standard_window_fns_pb2.py>
RefactoringTool: Files that were modified:
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_interactive_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_runner_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_runner_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/endpoints_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/external_transforms_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/metrics_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/schema_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/standard_window_fns_pb2.py>
INFO:gen_protos:Writing urn stubs: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/metrics_pb2_urns.py>
INFO:gen_protos:Writing urn stubs: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2_urns.py>
INFO:gen_protos:Writing urn stubs: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/standard_window_fns_pb2_urns.py>
INFO:gen_protos:Writing urn stubs: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2_urns.py>
INFO:gen_protos:Writing urn stubs: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2_urns.py>
INFO:gen_protos:Writing urn stubs: <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/portability/api/beam_runner_api_pb2_urns.py>
warning: no files found matching 'README.md'
warning: no files found matching 'NOTICE'
warning: no files found matching 'LICENSE'
warning: sdist: standard file not found: should have one of README, README.rst, README.txt, README.md
> Task :sdks:python:container:py37:copyDockerfileDependencies
> Task :runners:java-fn-execution:compileJava FROM-CACHE
> Task :runners:java-fn-execution:classes UP-TO-DATE
> Task :runners:java-fn-execution:jar
> Task :sdks:java:expansion-service:compileJava FROM-CACHE
> Task :sdks:java:expansion-service:classes UP-TO-DATE
> Task :sdks:java:expansion-service:jar
> Task :runners:java-job-service:compileJava FROM-CACHE
> Task :runners:java-job-service:classes UP-TO-DATE
> Task :runners:java-job-service:jar
> Task :runners:spark:compileJava FROM-CACHE
> Task :runners:spark:classes
> Task :runners:spark:jar
> Task :runners:spark:job-server:compileJava NO-SOURCE
> Task :runners:spark:job-server:classes UP-TO-DATE
> Task :runners:spark:job-server:shadowJar
> Task :sdks:go:resolveBuildDependencies
Resolving google.golang.org/api: commit='386d4e5f4f92f86e6aec85985761bba4b938a2d5', urls=[https://code.googlesource.com/google-api-go-client]
Resolving google.golang.org/genproto: commit='2b5a72b8730b0b16380010cfe5286c42108d88e7', urls=[https://github.com/google/go-genproto]
Resolving google.golang.org/grpc: commit='7646b5360d049a7ca31e9133315db43456f39e2e', urls=[https://github.com/grpc/grpc-go]
Resolving cached github.com/coreos/etcd: commit='11214aa33bf5a47d3d9d8dafe0f6b97237dfe921', urls=[https://github.com/coreos/etcd.git, git@github.com:coreos/etcd.git]
Resolving cached github.com/coreos/etcd: commit='11214aa33bf5a47d3d9d8dafe0f6b97237dfe921', urls=[https://github.com/coreos/etcd.git, git@github.com:coreos/etcd.git]
> Task :sdks:go:installDependencies
> Task :sdks:go:buildLinuxAmd64
> Task :sdks:go:goBuild
> Task :sdks:python:container:resolveBuildDependencies
Resolving ./github.com/apache/beam/sdks/go@<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/go>
> Task :sdks:python:container:installDependencies
> Task :sdks:python:container:buildDarwinAmd64
> Task :sdks:python:container:buildLinuxAmd64
> Task :sdks:python:container:goBuild
> Task :sdks:python:container:py37:copyLauncherDependencies
> Task :sdks:python:container:py37:dockerPrepare
> Task :sdks:python:container:py37:docker
The command '/bin/sh -c pip install -r /tmp/base_image_requirements.txt && python -c "from google.protobuf.internal import api_implementation; assert api_implementation._default_implementation_type == 'cpp'; print ('Verified fast protobuf used.')" && rm -rf /root/.cache/pip' returned a non-zero code: 1
> Task :sdks:python:container:py37:docker FAILED
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:container:py37:docker'.
> Process 'command 'docker'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 5m 0s
66 actionable tasks: 50 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/pv2yusqmqdydw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1370
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1370/display/redirect>
Changes:
------------------------------------------
[...truncated 65.27 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, chardet, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588939737.691574412","description":"Error received from peer ipv4:127.0.0.1:34685","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0508120857-85938732_8b801590-9c80-4bb3-906c-05cb97d51eac failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 39s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/7hbzi4a4mtrlu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1369
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1369/display/redirect?page=changes>
Changes:
[robertwb] Move job server to its own module.
[robertwb] [BEAM-9845] Stage artifacts over expansion service.
[robertwb] [BEAM-9577] Resolve dependencies in Java expansion service.
[robertwb] Start artifact service in expansion service driver. 0;256;0c# Please
[amaliujia] [BEAM-9929] Support UNNEST(array_column) in ZetaSQL.
------------------------------------------
[...truncated 66.25 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, urllib3, chardet, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 317, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 507, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588918244.616712592","description":"Error received from peer ipv4:127.0.0.1:41519","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0508061044-ed7750ff_08442218-8af5-435e-8faf-8d2137088600 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 25s
67 actionable tasks: 56 executed, 10 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/gttjmsf7qyhrm
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1368
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1368/display/redirect?page=changes>
Changes:
[github] Merge pull request #11624 from [BEAM-9767] Make streaming_wordcount use
[github] [BEAM-9731] passert.Equals: sort output strings for easier reading
[github] Revert "Pass pipeline options from caller to expansion service (#11574)"
[klk] Remove tests of Java on Dataflows Java Runner Harness
[github] [Beam-9679] Add Core Transforms section / Map lesson to the Go SDK katas
------------------------------------------
[...truncated 64.50 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/50/24/4d/4580ca4a299f1ad6fd63443e6e584cb21e9a07988e4aa8daac/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/59/b1/91/f02e76c732915c4015ab4010f3015469866c1eb9b14058d8e7/dill-0.3.1.1-cp37-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/8b/99/a0/81daf51dcd359a9377b110a8a886b3895921802d2fc1b2397e/future-0.18.2-cp37-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/fe/a7/05/23e3699975fc20f8a30e00ac1e515ab8c61168e982abe4ce70/hdfs-2.5.8-cp37-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/48/f7/87/b932f09c6335dbcf45d916937105a372ab14f353a9ca431d7d/oauth2client-3.0.0-cp37-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/5a/d3/be/86620c9dd3fca68986c33b9c616510289fc0abb81ec9aa70bd/avro_python3-1.9.2.1-cp37-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, chardet, certifi, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588896580.101717788","description":"Error received from peer ipv4:127.0.0.1:37965","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0508000940-126f25ea_ae97a721-462a-4182-960c-3123c3a928ca failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 19s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/rjlpuwdie3edw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1367
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1367/display/redirect?page=changes>
Changes:
[mxm] [BEAM-8742] Add stateful and timely processing benchmarks
[mxm] [BEAM-9900] Remove shutdownSourcesOnFinalWatermark flag
[github] [BEAM-9913] remove license dir if exists (#11630)
------------------------------------------
[...truncated 64.25 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, idna, urllib3, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 187, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 205, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588875003.857546520","description":"Error received from peer ipv4:127.0.0.1:44479","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 576, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0507181004-fcc909e0_3bbc5b55-5da7-49c8-9c2e-c5dc99c8e4c1 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 23s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/g7tqoo462f6wk
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1366
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1366/display/redirect?page=changes>
Changes:
[lcwik] [BEAM-8944] Improve UnboundedThreadPoolExecutor performance
[lcwik] fixup! Fix lint
[github] [BEAM-8134] Grafana dashboards for Load Tests and IO IT Performance
[github] [BEAM-8133] Push ioit tests metrics to influxdb
------------------------------------------
[...truncated 64.28 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/50/24/4d/4580ca4a299f1ad6fd63443e6e584cb21e9a07988e4aa8daac/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/59/b1/91/f02e76c732915c4015ab4010f3015469866c1eb9b14058d8e7/dill-0.3.1.1-cp37-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/8b/99/a0/81daf51dcd359a9377b110a8a886b3895921802d2fc1b2397e/future-0.18.2-cp37-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/fe/a7/05/23e3699975fc20f8a30e00ac1e515ab8c61168e982abe4ce70/hdfs-2.5.8-cp37-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/48/f7/87/b932f09c6335dbcf45d916937105a372ab14f353a9ca431d7d/oauth2client-3.0.0-cp37-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/5a/d3/be/86620c9dd3fca68986c33b9c616510289fc0abb81ec9aa70bd/avro_python3-1.9.2.1-cp37-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, idna, urllib3, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 187, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 205, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588854560.005291636","description":"Error received from peer ipv4:127.0.0.1:38161","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 576, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0507122920-f80e73bf_bbfca5a5-9ee7-4ad5-9a1e-80e767b20a8d failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 52s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/tsvslefbxgju6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1365
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1365/display/redirect?page=changes>
Changes:
[robertwb] Fix thread local to be initialized on every thread.
[github] un-duplicate _BEAM_SERVICES
[github] Remove a bunch of spurious warnings in tests. (#11621)
------------------------------------------
[...truncated 64.50 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, certifi, idna, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 187, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 205, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588831813.874729409","description":"Error received from peer ipv4:127.0.0.1:35637","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 576, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0507061014-ae1414da_7e714be5-ee05-4e0e-95a4-249a5e21b3e2 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
>>> FAILURE
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 54s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/trjuyg27eu4x2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1364
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1364/display/redirect?page=changes>
Changes:
[robertwb] [BEAM-7746] Enable mypy type checking for Beam Python code.
[kcweaver] [BEAM-9908] Fix Python build failures in release script.
[github] [BEAM-9840] Support for Parameterized Types when converting from
[kcweaver] Update instructions in release guide.
[iemejia] [BEAM-9836] Exclude Spark runner from UsesKeyInParDo tests
[github] Update sdks/python/mypy.ini
------------------------------------------
[...truncated 64.49 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/50/24/4d/4580ca4a299f1ad6fd63443e6e584cb21e9a07988e4aa8daac/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/59/b1/91/f02e76c732915c4015ab4010f3015469866c1eb9b14058d8e7/dill-0.3.1.1-cp37-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/8b/99/a0/81daf51dcd359a9377b110a8a886b3895921802d2fc1b2397e/future-0.18.2-cp37-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/fe/a7/05/23e3699975fc20f8a30e00ac1e515ab8c61168e982abe4ce70/hdfs-2.5.8-cp37-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/48/f7/87/b932f09c6335dbcf45d916937105a372ab14f353a9ca431d7d/oauth2client-3.0.0-cp37-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/5a/d3/be/86620c9dd3fca68986c33b9c616510289fc0abb81ec9aa70bd/avro_python3-1.9.2.1-cp37-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 187, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 205, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588810323.283675585","description":"Error received from peer ipv4:127.0.0.1:37443","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 576, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0507001203-a82e32dc_4c56f0ca-2868-4ad6-bfc5-e99854525eaa failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 161
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 50s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/exoadlqrtljwq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1363
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1363/display/redirect>
Changes:
------------------------------------------
[...truncated 64.10 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, chardet, certifi, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 187, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 205, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588788853.924530162","description":"Error received from peer ipv4:127.0.0.1:33999","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 576, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0506181414-7e7c8c7e_d7783df1-f313-4214-a6c8-f743c8a89809 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 31s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/fuh43kbxjrxte
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1362
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1362/display/redirect?page=changes>
Changes:
[je.ik] Disable two flaky tests (BEAM-8035, BEAM-9164)
[je.ik] [BEAM-9888] Drop data based on input watermark in
------------------------------------------
[...truncated 64.33 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, idna, urllib3, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 187, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 205, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588766960.220513122","description":"Error received from peer ipv4:127.0.0.1:40841","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 576, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0506120920-ba73f3cc_6ab2762d-9a8e-4091-9eec-87c7bcb2667e failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 0s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/gfaiymztpdwj4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1361
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1361/display/redirect>
Changes:
------------------------------------------
[...truncated 64.29 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, chardet, urllib3, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 187, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 205, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588745359.575068368","description":"Error received from peer ipv4:127.0.0.1:34157","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 576, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0506060919-bb9766ef_6663ccee-f470-4b07-bb58-6f2017351a1d failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 1s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/5vh4mldbpzare
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1360
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1360/display/redirect?page=changes>
Changes:
[github] Pass pipeline options from caller to expansion service (#11574)
------------------------------------------
[...truncated 64.04 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, idna, chardet, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 187, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 205, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588723924.278889627","description":"Error received from peer ipv4:127.0.0.1:46071","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 576, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0506001204-3819a5d_5b81fea5-7230-4968-81c3-acd8b2e2a741 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
>>> FAILURE
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 45s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/x4r4ohpnlwceq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1359
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1359/display/redirect?page=changes>
Changes:
[apilloud] [BEAM-9661] Fix ORDER BY with LIMIT
[apilloud] [BEAM-9659][BEAM-9657][BEAM-9664] Reject unsupported unnest joins
[github] [BEAM-9650] Add PeriodicSequence generator. (#11477)
------------------------------------------
[...truncated 64.49 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, idna, urllib3, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588702402.885032850","description":"Error received from peer ipv4:127.0.0.1:45901","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0505181323-c633b1e1_cbe58a3c-8d71-490f-9d4b-ba7c989d41fe failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 56s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/re3wwum6dlsrm
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1358
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1358/display/redirect?page=changes>
Changes:
[aiyangar] Changed sns writes to async
[pawel.pasterz] Update error prone
[pawel.pasterz] Aling EP core and annotations versions
[aiyangar] [BEAM-8542] V2 async changes for SNSIO
------------------------------------------
[...truncated 64.09 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, idna, chardet, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588681093.412430773","description":"Error received from peer ipv4:127.0.0.1:33787","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0505121813-4b7b121_01c0788d-4535-4a26-8e0a-912fd21f0391 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
>>> FAILURE
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 24s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/w6sqjezsfrcg2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1357
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1357/display/redirect?page=changes>
Changes:
[github] Merge pull request #11560 from [BEAM-9886] Auto-inferring project for
------------------------------------------
[...truncated 64.50 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, idna, chardet, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588658949.844619719","description":"Error received from peer ipv4:127.0.0.1:45145","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0505060910-cc5ed5a6_4094f168-f547-4a6f-a6b2-c6e194070294 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 50s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ugggi6qdbm3ce
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1356
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1356/display/redirect?page=changes>
Changes:
[jhnmora000] [BEAM-9418] Support ANY_VALUE aggregation functions
[tudor.marian] Do not attempt to estimate sizes of group-by-key values behind a key;
[tudor.marian] remove comments
[tudor.marian] Make BatchGroupAlsoByWindowViaIteratorsFn extend the
[tudor.marian] revert changes to grouping shuffle reader and iterable like coder
[tudor.marian] fix merge conflicts
[tudor.marian] remove obsolete class
[daniel.o.programmer] [BEAM-9799] Automated validation for RTrackers.
[daniel.o.programmer] [BEAM-9643] Adding Go SDF Documentation.
[github] [BEAM-9880] fix cannot touch issue (#11606)
[daniel.o.programmer] [BEAM-9643] Adjusting signature for TrySplit.
------------------------------------------
[...truncated 64.07 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588637399.167791743","description":"Error received from peer ipv4:127.0.0.1:38949","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0505000959-5593fd9c_d6d739a0-f23c-49d8-b90e-4d1f4c07c5d3 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 58s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/3wujamnaivhri
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1355
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1355/display/redirect?page=changes>
Changes:
[github] [BEAM-9870] Always generate Dataflow-compatible FnApi protos. (#11593)
[github] Go fmt changes. (#11576)
------------------------------------------
[...truncated 63.79 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, chardet, certifi, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588615760.086679701","description":"Error received from peer ipv4:127.0.0.1:41321","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0504180920-f5f53285_4719ef8a-8251-43d7-90f1-2b0a32f14e7a failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 58s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/w7vobbdcpbkcs
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1354
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1354/display/redirect?page=changes>
Changes:
[echauchot] [BEAM-8025] Increase the number of retrials en retrial delay in case of
[mxm] [BEAM-9841] Support finite wait on PortableRunner's PipelineResult
[mxm] [BEAM-9874] Support clearing timers in portable batch mode (Spark/Flink)
[echauchot] [BEAM-8025] Trace caught NoHostAvailableException and
------------------------------------------
[...truncated 64.76 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, chardet, certifi, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 194, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 212, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588594164.549136457","description":"Error received from peer ipv4:127.0.0.1:36609","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0504120924-3dd5ca29_37b38ff3-eaa1-4d59-96ed-5e44917f9b85 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 5s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ggc56acgjdf4s
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1353
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1353/display/redirect>
Changes:
------------------------------------------
[...truncated 64.07 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, certifi, idna, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588572539.809366765","description":"Error received from peer ipv4:127.0.0.1:34409","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 568, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0504060900-fec5ddce_e385ddc1-16ef-4d94-8e54-3987cd7a7c19 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 40s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ljmomcef6zfey
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1352
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1352/display/redirect>
Changes:
------------------------------------------
[...truncated 64.07 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, idna, chardet, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588550945.504195648","description":"Error received from peer ipv4:127.0.0.1:42489","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 568, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0504000905-3b50736b_3efdc398-7ea5-43a2-aaba-361f6c53670b failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 44s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/5ob2p354lo4oy
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1351
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1351/display/redirect>
Changes:
------------------------------------------
[...truncated 64.30 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Downloading numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, certifi, urllib3, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588529627.980381855","description":"Error received from peer ipv4:127.0.0.1:45629","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 568, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0503181348-cd92ce20_a852ce4f-153a-4de2-9d83-3e7ed08f6f7d failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 44s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/5uxxry364fdaw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1350
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1350/display/redirect?page=changes>
Changes:
[kcweaver] Refactor timer firing in Flink & Spark portable batch.
[kcweaver] [BEAM-9801] Fire timers set within timers in Spark.
------------------------------------------
[...truncated 64.51 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, certifi, idna, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588507770.951143835","description":"Error received from peer ipv4:127.0.0.1:34215","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 568, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0503120931-bfea8cca_6918abbd-fa83-4a06-be78-f44d73d2d0cd failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 11s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/w7edbtbz3hy22
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1349
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1349/display/redirect>
Changes:
------------------------------------------
[...truncated 64.58 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, urllib3, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588486161.573938471","description":"Error received from peer ipv4:127.0.0.1:40299","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 568, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0503060921-c8401801_024a4a3d-feb9-4adf-8379-939009c2bd55 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 0s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/tz6zzjzgcjrle
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1348
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1348/display/redirect?page=changes>
Changes:
[github] Merge pull request #11523 from [BEAM-8414] Cleanup Python codebase to
------------------------------------------
[...truncated 64.35 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, chardet, idna, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588464562.513051388","description":"Error received from peer ipv4:127.0.0.1:39347","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 568, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0503000922-24f99ac9_cfa58434-208f-4aad-8985-cd2d3ea70661 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 1s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ah2okynybzgo4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1347
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1347/display/redirect>
Changes:
------------------------------------------
[...truncated 64.05 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, certifi, chardet, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588442974.232989347","description":"Error received from peer ipv4:127.0.0.1:33341","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 568, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0502180934-5475f195_53259553-a85b-4f2f-b095-560b6e446024 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 13s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/vz2d5s2lbbvbq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1346
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1346/display/redirect>
Changes:
------------------------------------------
[...truncated 64.29 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, certifi, urllib3, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588421513.090045162","description":"Error received from peer ipv4:127.0.0.1:42795","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 568, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0502121153-f58bf1a_7f5acd8f-7b4f-4bff-a830-50d6f1025c9a failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 47s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/6lx7gigqjj32w
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1345
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1345/display/redirect>
Changes:
------------------------------------------
[...truncated 64.79 KB...]
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, chardet, urllib3, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:root:Waiting for grpc channel to be ready at localhost:36789.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588399781.464916148","description":"Error received from peer ipv4:127.0.0.1:46189","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 568, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0502060941-63b66bf_1d67fdbf-3c3a-44f5-b428-062ef1820536 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
>>> FAILURE
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 22s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/avsbrx5nu55v4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1344
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1344/display/redirect?page=changes>
Changes:
[kcweaver] [BEAM-9600] Get metrics in Flink uber jar job server.
[kcweaver] [BEAM-9600] Add a test for Flink REST metrics.
[rohde.samuel] [BEAM-9692] Replace the apply_read with a PTransformOverride
[bhulette] remove PipelineRunner.fromOptions call
[mxm] [BEAM-2128] Remove dependency on PipelineRunner#fromOptions to stage
[bhulette] Modify PipelineTest to check if exception is thrown on Pipeline run
[bhulette] Call FileSystems.setDefaultPipelineOptions in
[rohde.samuel] move redundant testing pieces to function
[mxm] [BEAM-9801] Pass in fire timestamp and pane info to timer callback
[mxm] [BEAM-9733] Repeatedly fire in batch mode until there are no more timers
[kcweaver] [BEAM-9860] Require job_endpoint when using PortableRunner.
[kcweaver] Replace portableWordCount tests with portableWordCountFlinkRunner.
[robertwb] Test simplification.
------------------------------------------
[...truncated 64.48 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, chardet, urllib3, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588378283.254102091","description":"Error received from peer ipv4:127.0.0.1:41073","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 568, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0502001123-7de44625_ac6d2c49-6533-49be-b383-e3a0f53797ad failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
>>> FAILURE
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 22s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ifz5w6c5u4qmo
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1343
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1343/display/redirect?page=changes>
Changes:
[robertwb] Guard grpc import.
------------------------------------------
[...truncated 63.82 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, certifi, urllib3, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588356762.970168632","description":"Error received from peer ipv4:127.0.0.1:43181","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0501181243-eeeb79d9_98406cc5-d663-4a25-8710-4640bca22457 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 58s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/liaogqtzhxl62
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1342
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1342/display/redirect>
Changes:
------------------------------------------
[...truncated 64.51 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, certifi, chardet, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588334983.139319344","description":"Error received from peer ipv4:127.0.0.1:46767","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0501120943-9fe4e33_c1a0a6ff-31f8-4b10-a18d-6b6835f5076e failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 23s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ttl3qn2e6qfnu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1341
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1341/display/redirect>
Changes:
------------------------------------------
[...truncated 63.82 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, chardet, certifi, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588313499.659831403","description":"Error received from peer ipv4:127.0.0.1:39919","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0501061139-3848873a_1f4af9b3-42c3-4843-bae6-cf574ce1e6df failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 32s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/grsg2udxtsxjq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1340
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1340/display/redirect?page=changes>
Changes:
[sniemitz] [BEAM-9795] Support custom avro DatumWriters when writing to BigQuery
[github] skip pulling licenses by default (#11548)
[boyuanz] Update container version
[github] [BEAM-8078] streaming_wordcount_debugging.py is missing a test (#11568)
[github] Merge pull request #11538 from [BEAM-9831] Improve UX for HL7v2IO
------------------------------------------
[...truncated 64.26 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, chardet, urllib3, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588292040.748060014","description":"Error received from peer ipv4:127.0.0.1:39459","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0501001401-270dc8a0_f4734e1d-6ce7-4eb8-9a50-40d808971ffe failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 8s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/x32p35hhyt3ik
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1339
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1339/display/redirect?page=changes>
Changes:
[echauchot] [BEAM-5192] move metadata serialization configuration code to
[github] [BEAM-9720] Add Aws Http client configurations capability. (#11341)
------------------------------------------
[...truncated 64.08 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, urllib3, idna, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588270565.503307688","description":"Error received from peer ipv4:127.0.0.1:34305","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0430181605-c81d4eb0_446dc98a-066a-46c1-9c5e-e5282ca75c8e failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 29s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/46wpqz42cum7e
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1338
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1338/display/redirect>
Changes:
------------------------------------------
[...truncated 64.27 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, chardet, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588248581.985143430","description":"Error received from peer ipv4:127.0.0.1:35669","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0430120942-fd157eaf_802e131e-4066-4be8-85a9-64d58038abed failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 22s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/veogybznxh5pi
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1337
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1337/display/redirect?page=changes>
Changes:
[alxavier] [BEAM-9739]Retry SpannerIO write on Schema change
[github] [BEAM-9561] Add a framework for running pandas doctests with beam
------------------------------------------
[...truncated 64.51 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, urllib3, certifi, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588226943.119683948","description":"Error received from peer ipv4:127.0.0.1:40239","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0430060903-3b958879_f6b19a99-5013-4f7a-adb5-138575bbf276 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 43s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/33kou5nfo73mg
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1336
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1336/display/redirect?page=changes>
Changes:
[lcwik] [BEAM-9846] Clean up references to unused native BQ source/sink
[dcavazos] [BEAM-9771] Fix notebook github URLs
[github] Fix typo in go installation link. (#11445)
------------------------------------------
[...truncated 64.02 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, urllib3, idna, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588205363.018929953","description":"Error received from peer ipv4:127.0.0.1:37505","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0430000923-2d29b7a3_5f3e4fbb-f87c-4dfa-8b09-c425f3bd4111 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 3s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/7b3hv7uei4z4g
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1335
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1335/display/redirect?page=changes>
Changes:
[robertwb] [9577] Attempt to stage artifacts via new API on Java.
[robertwb] [BEAM-9802] Customization of automatically started services.
------------------------------------------
[...truncated 64.27 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Downloading fastavro-0.23.3-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, urllib3, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.3 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588183781.763757490","description":"Error received from peer ipv4:127.0.0.1:40807","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0429180942-2cbbba77_47d98fa4-562c-4ff9-a5cc-1d6b5de09d07 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 21s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/umcikowzgxg7o
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1334
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1334/display/redirect?page=changes>
Changes:
[github] [BEAM-9147] Make VideoIntelligence use PTransform on user-facing API
------------------------------------------
[...truncated 63.80 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, certifi, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588162178.006075470","description":"Error received from peer ipv4:127.0.0.1:41555","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0429120938-8645cbcf_71802f25-8a61-474b-bf4e-b51ba77cc500 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
>>> FAILURE
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 17s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/zb6e5vghjdwvq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1333
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1333/display/redirect>
Changes:
------------------------------------------
[...truncated 64.04 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, certifi, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588140592.618092135","description":"Error received from peer ipv4:127.0.0.1:33611","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0429060952-559893c4_87de1194-1d85-4860-924a-53c944237296 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 33s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/fqolawaclpzca
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1332
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1332/display/redirect?page=changes>
Changes:
[robertwb] Conditionally cleanup tempdir.
[mxm] [BEAM-6661] Properly close channel for external environment
[mxm] [BEAM-6661] Avoid FileNotFoundException when no files have been staged
[mxm] [BEAM-6661] Avoid printing empty lines in subprocess job server
[robertwb] Use logging to reduce verbosity of license checking when all goes well.
[github] Speedup on FnApiRunner start up time (#11543)
[github] All Python DirectRunners call setup/teardown on DoFns (#11547)
[github] [BEAM-9815] Correct go integration test if clauses. (#11524)
[relax] Support schemas in state API
[github] [BEAM-7923] Support docker screendiff integration tests (#11507)
------------------------------------------
[...truncated 64.03 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, certifi, chardet, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588119410.173230096","description":"Error received from peer ipv4:127.0.0.1:33489","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0429001650-533638c9_a4ee8c85-f320-48da-9db2-461ffcedeb2b failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 24s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ysjr5b6uvwf2a
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1331
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1331/display/redirect>
Changes:
------------------------------------------
[...truncated 63.79 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, idna, certifi, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588102683.199091112","description":"Error received from peer ipv4:127.0.0.1:36159","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0428193803-ce269ab_7e0345d1-cb52-46d0-a541-b75eea27ba3a failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 40s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/iunffcfau3bio
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1330
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1330/display/redirect?page=changes>
Changes:
[github] Display correct package in help message
[apilloud] [BEAM-9658] Plumb through WITH OFFSET
------------------------------------------
[...truncated 64.55 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, certifi, urllib3, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588096921.493891684","description":"Error received from peer ipv4:127.0.0.1:41995","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0428180201-1c01b7f2_0a22d8c3-59e4-484a-9375-0b82099da3a6 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 17m 8s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/fsrkc47envzkc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1329
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1329/display/redirect?page=changes>
Changes:
[github] [BEAM-9443] update document about direct_num_workers=0 for FnApi
------------------------------------------
[...truncated 64.30 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, chardet, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588092633.600744861","description":"Error received from peer ipv4:127.0.0.1:37653","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0428165033-628cc606_73baa438-3a80-4a38-a70b-370db7ab9f74 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 59s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/kj652e3jh5afk
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1328
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1328/display/redirect?page=changes>
Changes:
[kcweaver] [BEAM-9758] Use quotes in echo statements to prevent * expansion.
------------------------------------------
[...truncated 64.04 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588086560.532034500","description":"Error received from peer ipv4:127.0.0.1:33725","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0428150920-b5b0f1dd_3c970d31-b430-48f0-a9af-2306d0eabb63 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 0s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/7mj6lqgm457zo
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1327
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1327/display/redirect>
Changes:
------------------------------------------
[...truncated 64.04 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Downloading pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, urllib3, chardet, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588077711.973558793","description":"Error received from peer ipv4:127.0.0.1:41053","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0428124152-4f10e068_310975ae-60a6-422c-8627-d47d3c33bd96 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 19s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/2or5tj5jwdnyu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1326
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1326/display/redirect?page=changes>
Changes:
[kamil.wasilewski] [BEAM-8790] Kubernetes deployment for InfluxDB
[kamil.wasilewski] [BEAM-8790] Local deployment of InfluxDB with docker-compose.yml
------------------------------------------
[...truncated 63.82 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Downloading pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, chardet, certifi, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588075438.708468314","description":"Error received from peer ipv4:127.0.0.1:36283","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0428120358-944422f7_eaee28c3-ccc1-43ef-bc26-b0f94593f0ef failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 40s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/6qsx4xy6rd4dw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1325
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1325/display/redirect?page=changes>
Changes:
[david.moravek] [BEAM-9824] Do not ignore chained Reshuffles on flink batch runner.
------------------------------------------
[...truncated 64.31 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, urllib3, certifi, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588059805.201161361","description":"Error received from peer ipv4:127.0.0.1:38027","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0428074325-c80c0772_c5195790-1ae8-4e36-85c2-7403e4597d2d failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
>>> FAILURE
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 38s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/w3mjr7vb446hq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1324
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1324/display/redirect>
Changes:
------------------------------------------
[...truncated 64.51 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, certifi, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588054160.431562133","description":"Error received from peer ipv4:127.0.0.1:37783","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0428060920-148c29b9_e9d9565e-7c6e-4a80-8cc9-f6deecd67846 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
>>> FAILURE
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 56s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/odblkbpd3hlys
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1323
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1323/display/redirect?page=changes>
Changes:
[github] Merge pull request #11546 from [BEAM-9832] Ensuring tags are not added
------------------------------------------
[...truncated 64.33 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, chardet, certifi, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588048481.671053364","description":"Error received from peer ipv4:127.0.0.1:33221","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0428043441-97c34be0_6e51f438-011b-4648-9369-dab7bbf70e1b failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 4s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/zroeecof2k3cc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1322
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1322/display/redirect>
Changes:
------------------------------------------
[...truncated 64.02 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, chardet, urllib3, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588032857.613086939","description":"Error received from peer ipv4:127.0.0.1:34803","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0428001417-1e8bba2b_e0019655-393c-45dd-a0e0-90d387d05e7d failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 21s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/xilyz3r5tza5w
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1321
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1321/display/redirect?page=changes>
Changes:
[kcweaver] [BEAM-9833] Add .asf.yaml file.
[kcweaver] Send issues to github@
[kcweaver] Add license header.
------------------------------------------
[...truncated 64.27 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, certifi, urllib3, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588029903.395192856","description":"Error received from peer ipv4:127.0.0.1:34881","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0427232503-f26b1416_95a7c2eb-ff18-4691-9582-a580fd9b606e failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 19s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/wlrvkdlhbqlay
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1320
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1320/display/redirect?page=changes>
Changes:
[kcweaver] [BEAM-9807] [BEAM-9808] Add missing region to Dataflow examples.
[kcweaver] Missed a few spots.
[kcweaver] Only add --region for Dataflow runner.
[bhulette] Fix JavaSDK typo
[bhulette] Add Python SqlTransform to CHANGES.md
[github] virtualenv (#11502)
------------------------------------------
[...truncated 605.92 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, chardet, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588026628.860370749","description":"Error received from peer ipv4:127.0.0.1:34251","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0427223029-1f1db37c_f0308c90-4b0f-415d-b8f5-a99e82197587 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 48s
65 actionable tasks: 63 executed, 1 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/aopdon5leaelu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1319
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1319/display/redirect?page=changes>
Changes:
[github] [BEAM-9779] Patch HL7v2IOWriteIT Flakiness (#11450)
------------------------------------------
[...truncated 64.05 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, urllib3, certifi, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588025250.784354985","description":"Error received from peer ipv4:127.0.0.1:43825","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0427220731-26678037_d8844be7-b516-4c76-b08e-79b6151554a7 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 46s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/usapvt7wq355k
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1318
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1318/display/redirect?page=changes>
Changes:
[github] Updates Dataflow Python containers to 20200423 (#11535)
------------------------------------------
[...truncated 64.05 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, chardet, urllib3, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588024402.597100204","description":"Error received from peer ipv4:127.0.0.1:35125","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0427215322-3c26d42f_e4146943-3be7-439d-a6f6-d9bb9468ed82 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 48s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/65mtamcqsflay
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1317
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1317/display/redirect>
Changes:
------------------------------------------
[...truncated 64.05 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, idna, certifi, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588011211.879008509","description":"Error received from peer ipv4:127.0.0.1:44591","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0427181332-44284b0_8229e642-0695-4009-8652-f99bb80c462b failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
>>> FAILURE
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 52s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/anqeex3vifj5q
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1316
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1316/display/redirect?page=changes>
Changes:
[github] Merge pull request #11154: [BEAM-1819] Key should be available in
------------------------------------------
[...truncated 602.00 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, chardet, idna, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588006414.950007509","description":"Error received from peer ipv4:127.0.0.1:37145","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0427165335-e703ec12_befdb058-b195-409c-b091-9c0b98efdffe failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 26s
65 actionable tasks: 60 executed, 4 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/wngesd2633hz6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1315
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1315/display/redirect?page=changes>
Changes:
[mxm] [BEAM-9827] Ensure minimum watermark hold is computed across all keys
------------------------------------------
[...truncated 63.87 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, urllib3, chardet, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1588003977.396577153","description":"Error received from peer ipv4:127.0.0.1:34113","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0427161257-6113b021_6356306f-9c1a-4b68-82aa-0ae4188f8384 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
>>> FAILURE
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 31s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/4plv56msq4rwu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1314
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1314/display/redirect?page=changes>
Changes:
[coheigea] BEAM-9826 - Update Tika to 1.24.1
------------------------------------------
[...truncated 64.28 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, urllib3, idna, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587997567.105324062","description":"Error received from peer ipv4:127.0.0.1:35345","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0427142607-19ae5f88_07916911-6a21-48d3-8417-c4a291042b78 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 57s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/eanbba7vace4w
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1313
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1313/display/redirect>
Changes:
------------------------------------------
[...truncated 63.79 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, chardet, certifi, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587989338.361612445","description":"Error received from peer ipv4:127.0.0.1:40975","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0427120858-3799dadf_fd68671e-0ddb-4174-bf10-764a103308fc failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 39s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/545e33uckluto
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1312
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1312/display/redirect>
Changes:
------------------------------------------
[...truncated 64.18 KB...]
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Using cached httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, certifi, chardet, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:root:Waiting for grpc channel to be ready at localhost:58325.
WARNING:root:Waiting for grpc channel to be ready at localhost:58325.
WARNING:root:Waiting for grpc channel to be ready at localhost:58325.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587967901.449950687","description":"Error received from peer ipv4:127.0.0.1:42139","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0427061142-73ba93aa_819da1ff-4b2f-4c3a-851b-155a86a47bf3 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
>>> FAILURE
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 11m 25s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/6jzimn3b3qoxc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1311
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1311/display/redirect>
Changes:
------------------------------------------
[...truncated 63.79 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Downloading httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, certifi, idna, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587946144.904178978","description":"Error received from peer ipv4:127.0.0.1:37587","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0427000905-c64b9be9_e53bd4cd-478d-4899-9389-707a15f5bd43 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 45s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/wyhtsu7mzbcmm
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1310
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1310/display/redirect?page=changes>
Changes:
[iemejia] [BEAM-9820] Upgrade Flink 1.9.x to 1.9.3
[github] [BEAM-9819] Relaxes the version range for httplib2 to <0.16.0. (#11522)
------------------------------------------
[...truncated 63.94 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.16.0,>=0.8
Downloading httplib2-0.15.0-py3-none-any.whl (94 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, idna, chardet, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.15.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587935976.175355150","description":"Error received from peer ipv4:127.0.0.1:42703","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0426211936-b107bf9d_e8a59f5d-89f3-4250-b1da-fdd445df6706 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 12s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/lf57ehnleuol2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1309
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1309/display/redirect?page=changes>
Changes:
[github] Fix invalid formatting specifier in verify
------------------------------------------
[...truncated 63.54 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, chardet, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587935250.795170368","description":"Error received from peer ipv4:127.0.0.1:33623","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0426210731-988693b1_bbe02aa4-765d-4859-ba5a-ec027e547b48 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 44s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/2ujd4zht2hbgy
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1308
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1308/display/redirect>
Changes:
------------------------------------------
[...truncated 64.26 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, certifi, urllib3, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587924540.819945607","description":"Error received from peer ipv4:127.0.0.1:35873","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0426180901-99de3863_ac341493-ea8d-4f97-8c2f-ee08c1c75f30 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 41s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/a5kdiudux6vpi
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1307
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1307/display/redirect>
Changes:
------------------------------------------
[...truncated 64.27 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, certifi, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587902900.399610180","description":"Error received from peer ipv4:127.0.0.1:33743","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0426120820-95eca520_67c1eda7-868a-4e12-8ecd-52dcca0f79c5 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 1s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/poi6vp2trp5xy
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1306
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1306/display/redirect>
Changes:
------------------------------------------
[...truncated 64.24 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, urllib3, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587881317.978120741","description":"Error received from peer ipv4:127.0.0.1:34057","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0426060838-6abfad51_ec54a1af-37a5-4317-96aa-e866c3fe4f9a failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 19s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/oi5ks7zzy5nl6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1305
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1305/display/redirect?page=changes>
Changes:
[github] [BEAM-9488] Ensure we pass through PCollection ids instead of attempting
------------------------------------------
[...truncated 64.05 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, urllib3, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587877326.773238930","description":"Error received from peer ipv4:127.0.0.1:39211","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0426050207-b431225e_5fed749e-22d4-47b1-8088-ca2187703bab failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 54s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/36dtfonxp5lvc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1304
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1304/display/redirect>
Changes:
------------------------------------------
[...truncated 64.48 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, urllib3, chardet, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587859953.328562355","description":"Error received from peer ipv4:127.0.0.1:45175","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0426001233-42ea810_e7e38815-e84a-4558-b3ee-7ef23adf4fd4 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 11m 29s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/op3haoc37neau
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1303
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1303/display/redirect>
Changes:
------------------------------------------
[...truncated 64.50 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, chardet, idna, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587842505.601836756","description":"Error received from peer ipv4:127.0.0.1:43535","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0425192145-7d22ad77_32abbcd2-e242-4888-bbfc-144386e662a9 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 5s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/los33o4udzazo
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1302
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1302/display/redirect>
Changes:
------------------------------------------
[...truncated 63.79 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, idna, urllib3, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587794962.914796234","description":"Error received from peer ipv4:127.0.0.1:41399","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0425060923-2ae15045_a614c5f5-8a39-4220-a2eb-c028c6a96f4b failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 3s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/r3poeu7ip2uri
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1301
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1301/display/redirect>
Changes:
------------------------------------------
[...truncated 64.02 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, chardet, certifi, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587773372.581216556","description":"Error received from peer ipv4:127.0.0.1:45803","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0425000932-424346f1_eb828e14-0165-485d-b8ea-da0a97ea63d1 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 41s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/j4smwtjsqexd6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1300
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1300/display/redirect?page=changes>
Changes:
[github] add licenses and license types for zetasql packages (#11516)
[github] Added a batch example with covid tracking data for interactive notebook.
------------------------------------------
[...truncated 63.79 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, certifi, urllib3, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587751751.093612828","description":"Error received from peer ipv4:127.0.0.1:35725","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0424180911-4016d994_c0b0374e-95fb-47df-8ce7-adf73e094206 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 52s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/pqoafu3dqtyao
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1299
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1299/display/redirect?page=changes>
Changes:
[tweise] [BEAM-9811] Copy pom for all publishMavenJavaPublication tasks
------------------------------------------
[...truncated 63.81 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, certifi, urllib3, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587744595.913877618","description":"Error received from peer ipv4:127.0.0.1:33859","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0424160956-8769a88b_492e3ef7-1fca-444e-b16f-0225fb7d6824 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 7s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/k3i4gnyl363ii
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1298
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1298/display/redirect?page=changes>
Changes:
[kamil.wasilewski] [BEAM-9761] Disable auto builds for tests that are currently failing.
------------------------------------------
[...truncated 64.07 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, chardet, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587741844.508821244","description":"Error received from peer ipv4:127.0.0.1:41705","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0424152404-bd4cd7ce_78a4a920-386c-4cd7-a60f-ed804226591f failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 58s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/tm274klqfi3a4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1297
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1297/display/redirect?page=changes>
Changes:
[robertwb] [BEAM-9813] Replace outputs with respect to the correct tag.
------------------------------------------
[...truncated 63.82 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, chardet, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587739868.656177588","description":"Error received from peer ipv4:127.0.0.1:37241","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0424145108-bbfbf9cf_9500acba-a257-4732-9a4f-67432b554671 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 20s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/okucrhkg326fq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1296
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1296/display/redirect>
Changes:
------------------------------------------
[...truncated 64.22 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, idna, certifi, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587730120.743640843","description":"Error received from peer ipv4:127.0.0.1:39951","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0424120840-85bc4abf_f969df95-735f-4681-b369-776a630aae1b failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
>>> FAILURE
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 22s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/aixmuqye6x6gu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1295
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1295/display/redirect?page=changes>
Changes:
[mxm] [BEAM-9733] Always let ImpulseSourceFunction emit a final watermark
[mxm] [BEAM-9733] Make up for timers set while processing the bundle
------------------------------------------
[...truncated 64.28 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, chardet, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587725760.668395158","description":"Error received from peer ipv4:127.0.0.1:33233","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0424105600-e030f462_d5ad0852-2cb8-4f11-ac1b-04f3628d0c13 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 40s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/d5ak2g5o2mcus
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1294
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1294/display/redirect>
Changes:
------------------------------------------
[...truncated 64.25 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, chardet, urllib3, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587708668.756961469","description":"Error received from peer ipv4:127.0.0.1:36695","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0424061109-22c5b64b_f8d2f581-aaa7-41f8-bfe3-2c1b672d3228 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 16s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/i5f7l2j6ysgqm
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1293
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1293/display/redirect?page=changes>
Changes:
[github] [BEAM-9775] Adding Go SDF example, adjusting GetProgress signature.
------------------------------------------
[...truncated 64.03 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, certifi, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587703918.182373502","description":"Error received from peer ipv4:127.0.0.1:34323","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0424045158-c633b34e_b26760e9-6634-45eb-9033-92e7fd53e951 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 2s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/fhw476jrdiazg
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1292
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1292/display/redirect?page=changes>
Changes:
[tweise] [BEAM-9811] Nightly snapshot publish error
------------------------------------------
[...truncated 64.46 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, chardet, idna, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587702259.898425796","description":"Error received from peer ipv4:127.0.0.1:41661","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0424042420-5c843aa1_ad4c6e57-84d8-4775-927b-7af487c2d881 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 40s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/od5mblfq2d4q2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1291
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1291/display/redirect?page=changes>
Changes:
[boyuanz] [BEAM-8871] Support trySplit for ByteKeyRangeTracker
------------------------------------------
[...truncated 63.79 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, urllib3, chardet, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587694069.845758678","description":"Error received from peer ipv4:127.0.0.1:33619","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0424020750-c865ec27_515c64dc-e1a3-4f43-82f0-e991d025557a failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 1s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/lv57lr464edaq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1290
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1290/display/redirect?page=changes>
Changes:
[ehudm] [BEAM-7405] Workaround for bad Docker config
------------------------------------------
[...truncated 63.83 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, urllib3, idna, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587689971.716681463","description":"Error received from peer ipv4:127.0.0.1:39175","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0424005931-cc9f670e_cf26d0b3-386e-4840-98bb-c2595639103b failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 17s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/kdiawjdjwqlzu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1289
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1289/display/redirect>
Changes:
------------------------------------------
[...truncated 64.25 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, certifi, urllib3, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587687119.405921311","description":"Error received from peer ipv4:127.0.0.1:34385","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0424001159-a23c40dd_383acbd3-1443-4536-9f63-e1a66da5a8cf failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 28s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/bobdhpgiu45u2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1288
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1288/display/redirect?page=changes>
Changes:
[apilloud] [BEAM-9514] Ensure nullability passes through sum
------------------------------------------
[...truncated 64.49 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, idna, urllib3, certifi, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587679496.740927274","description":"Error received from peer ipv4:127.0.0.1:37899","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0423220456-426ab568_c3beea17-dee2-4e60-8f6c-452823510ca2 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 4s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/dow56btrcduwe
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1287
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1287/display/redirect?page=changes>
Changes:
[pabloem] [BEAM-9812] Fixing bug causing pipelines requiring temp tables to not
------------------------------------------
[...truncated 64.27 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, chardet, certifi, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587675380.138511562","description":"Error received from peer ipv4:127.0.0.1:42031","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0423205620-2eb6ad9a_8189b9b0-34ca-464b-9412-87f1f25e12a7 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 57s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/aeovuct2iicsi
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1286
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1286/display/redirect?page=changes>
Changes:
[robertwb] [BEAM-9577] New artifact staging and retrieval service for Java.
[robertwb] [BEAM-9577] Multi-threaded artifact staging service backend.
[robertwb] Use futures, better error handling.
------------------------------------------
[...truncated 64.88 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, certifi, chardet, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587673651.248209664","description":"Error received from peer ipv4:127.0.0.1:38551","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0423202731-4d0b831c_48f6998b-9a1a-4d04-801f-f27c7bbc87bb failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 18s
65 actionable tasks: 52 executed, 12 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/sydzupmef74um
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1285
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1285/display/redirect?page=changes>
Changes:
[lcwik] [BEAM-2939] Expose HasProgress interface for restriction trackers and
[lcwik] [BEAM-2939] Add the ability for SDK harness runners to provide
[lcwik] [BEAM-2939] Integrate progress reporting for splittable dofns.
[lcwik] [BEAM-2939] Fold Sizes sub-interfaces into RestrictionTracker
[lcwik] [BEAM-2939] Drop HasSize in favor of using
------------------------------------------
[...truncated 64.51 KB...]
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, chardet, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:root:Waiting for grpc channel to be ready at localhost:60657.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587669652.762948292","description":"Error received from peer ipv4:127.0.0.1:34697","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0423192053-5a9a8c09_71485ed3-22fc-4031-9f5d-db0cc7bd6682 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 13m 16s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/54bml5frza5iu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1284
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1284/display/redirect>
Changes:
------------------------------------------
[...truncated 64.26 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Downloading fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, chardet, urllib3, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587665512.257755420","description":"Error received from peer ipv4:127.0.0.1:34233","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0423181152-f430e36e_b3430919-45cc-4043-931b-9e6a159b59f5 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 33s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/islksrcvb7xss
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1283
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1283/display/redirect?page=changes>
Changes:
[github] [BEAM-9701] Increments fastavro version range upper bound to 0.24.
------------------------------------------
[...truncated 64.24 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Downloading fastavro-0.23.2-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, certifi, urllib3, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.2 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587662439.801669690","description":"Error received from peer ipv4:127.0.0.1:38275","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0423172040-78f16d88_fffa6c3c-73d4-49da-8869-fe3a6552d714 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 12s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/pmzm73dol6l3g
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1282
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1282/display/redirect?page=changes>
Changes:
[mxm] [BEAM-9794] Reduce state cells needed for BufferingDoFnRunner
------------------------------------------
[...truncated 64.01 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, certifi, idna, chardet, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587653204.310736123","description":"Error received from peer ipv4:127.0.0.1:33743","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0423144644-2c1fe602_91affd81-8b24-4af5-83e8-2b11401dff8c failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 32s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/tgmxllwmwins6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1281
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1281/display/redirect>
Changes:
------------------------------------------
[...truncated 64.42 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, chardet, idna, certifi, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587643794.533801757","description":"Error received from peer ipv4:127.0.0.1:45393","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0423120954-61ce6b24_db481a76-2e09-448c-aa13-109373a96587 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 38s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/3kkg72us3womw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1280
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1280/display/redirect>
Changes:
------------------------------------------
[...truncated 63.95 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, chardet, certifi, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587622162.562799996","description":"Error received from peer ipv4:127.0.0.1:45267","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0423060922-fad51a41_ebd391b8-cff1-4b08-8e02-695084334562 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 2s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/pusxbhosu3evo
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1279
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1279/display/redirect?page=changes>
Changes:
[github] [BEAM-8603] Add Python SqlTransform (#10055)
------------------------------------------
[...truncated 64.46 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, certifi, chardet, urllib3, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587600685.290048955","description":"Error received from peer ipv4:127.0.0.1:38659","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0423001125-42c6dbb3_8c170f56-1b39-4f08-bf32-bcd49cb27c13 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 30s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/6as3opc2m6wdy
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1278
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1278/display/redirect?page=changes>
Changes:
[github] [BEAM-9778] remove temp directory & add docker-pull-licenses to release
------------------------------------------
[...truncated 64.31 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, certifi, chardet, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587594575.944878291","description":"Error received from peer ipv4:127.0.0.1:36911","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0422222936-74b39ef5_14a41d10-6210-4578-80da-aefae774d5a4 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 4s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/3q653aqubwldc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1277
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1277/display/redirect?page=changes>
Changes:
[github] Force output names to be unicode-compatible on Python 2. (#11485)
[github] Make license_script.sh tolerate directory already existing (#11501)
------------------------------------------
[...truncated 63.83 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, certifi, urllib3, chardet, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587592746.608175348","description":"Error received from peer ipv4:127.0.0.1:33287","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0422215906-5ae2bdcc_20238f94-e575-4568-9b2c-2e3780e27e06 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 49s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/tslhpd72potkg
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1276
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1276/display/redirect?page=changes>
Changes:
[github] [BEAM-9577] Use new artifact API to stage go artifacts. (#11490)
------------------------------------------
[...truncated 63.81 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, urllib3, idna, certifi, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587588444.483329190","description":"Error received from peer ipv4:127.0.0.1:37765","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0422204724-fb4b405c_2e8f4dbf-c928-44a6-9449-8e4fc6a13394 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 44s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/3gvtkknsnykqk
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1275
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1275/display/redirect?page=changes>
Changes:
[github] [BEAM-9717] update future version to 0.18.2 for Python 3.8 (#11494)
------------------------------------------
[...truncated 64.25 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, chardet, idna, urllib3, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587585585.654143570","description":"Error received from peer ipv4:127.0.0.1:41611","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0422195945-c508cebb_df6f06f3-c16c-4936-88dd-927826830c73 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 27s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/gadi5lrfa22bc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1274
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1274/display/redirect?page=changes>
Changes:
[yoshiki.obata] [BEAM-9755] changed MAKE_ADD stack pop order according to Python version
[github] [BEAM-9642] Enabling SDF exec runners and fixing small bug. (#11474)
------------------------------------------
[...truncated 64.33 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, chardet, certifi, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587579849.564323120","description":"Error received from peer ipv4:127.0.0.1:42945","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0422182410-eba744b_32982af5-bda3-497a-8653-b86567fa57ee failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
>>> FAILURE
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 13m 10s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/cmevde75stlcs
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1273
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1273/display/redirect?page=changes>
Changes:
[github] Merge pull request #11489 [BEAM-9577] Ensure required directories exist
------------------------------------------
[...truncated 64.04 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, chardet, idna, certifi, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587576856.437942313","description":"Error received from peer ipv4:127.0.0.1:41993","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0422173416-b36a3ff1_cf221025-eb67-464a-b470-501e105d8363 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 55s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/on3llvi4dg4uw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1272
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1272/display/redirect?page=changes>
Changes:
[markliu] [BEAM-8831] Do not copy generated files for Python source copy
------------------------------------------
[...truncated 64.48 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, urllib3, idna, chardet, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587574464.496431033","description":"Error received from peer ipv4:127.0.0.1:41759","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0422165424-74d3e748_d35e6da7-2dd4-494e-a560-34efa6aec282 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 34s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/k4v42zs36w4sk
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1271
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1271/display/redirect?page=changes>
Changes:
[github] [BEAM-6597, BEAM-6467] Add support for reporting monitoring infos to the
------------------------------------------
[...truncated 64.49 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, idna, urllib3, chardet, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587567013.362911374","description":"Error received from peer ipv4:127.0.0.1:40615","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0422145013-f9f83601_e39b7164-f125-4d97-a78a-923d1588f6fe failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 17s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/jmphk4szpxvpk
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1270
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1270/display/redirect>
Changes:
------------------------------------------
[...truncated 63.76 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, chardet, idna, urllib3, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587557334.769959535","description":"Error received from peer ipv4:127.0.0.1:42489","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0422120855-56c5f708_90c242cb-3a52-4746-8dca-347466bf9989 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 36s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/iyg2llf2e6bz4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1269
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1269/display/redirect>
Changes:
------------------------------------------
[...truncated 64.22 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, urllib3, chardet, idna, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587535733.095643358","description":"Error received from peer ipv4:127.0.0.1:45421","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0422060853-6323884f_50d4a294-6cab-4623-8a73-59fc97801c44 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 33s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/jvkz7y3wszddm
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1268
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1268/display/redirect?page=changes>
Changes:
[github] Log job id similar to Java and Go
------------------------------------------
[...truncated 64.06 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, urllib3, idna, chardet, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587519780.822674922","description":"Error received from peer ipv4:127.0.0.1:32797","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0422014301-10d6393a_f41f74ca-430f-4396-95c9-4f48d08466d9 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
>>> FAILURE
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 58s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/lgym5d2vorn7m
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1267
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1267/display/redirect?page=changes>
Changes:
[shoaib.zafar] update release 2.20.0 blog post and added contributors name
------------------------------------------
[...truncated 64.05 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, idna, chardet, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587516243.419960995","description":"Error received from peer ipv4:127.0.0.1:33943","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0422004403-1906f124_b1e12180-0050-4815-a43c-398a9275e391 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 34s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/f677w23uu36es
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1266
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1266/display/redirect?page=changes>
Changes:
[github] [BEAM-7981] Correct ParDo(callable) test (#11446)
------------------------------------------
[...truncated 63.79 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, urllib3, certifi, chardet, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587514454.738033703","description":"Error received from peer ipv4:127.0.0.1:40133","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0422001414-c4c76522_1320d5f8-9496-4d17-921e-50ff1cd00c0e failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 1s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/o3erpiloom75s
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1265
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1265/display/redirect?page=changes>
Changes:
[rohde.samuel] Add a timeout to the TestStream GRPC and fix the Streaming cache timeout
------------------------------------------
[...truncated 64.30 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, certifi, chardet, idna, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587510808.962611975","description":"Error received from peer ipv4:127.0.0.1:39965","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0421231329-1aa84deb_b2615df1-9c97-4de3-bde4-9880c0d86242 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 35s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/dgagexs6fs5wg
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1264
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1264/display/redirect?page=changes>
Changes:
[github] Revert "[BEAM-9014] CachingShuffleBatchReader use bytes to limit cache
[github] [BEAM-9789] Fix lock error. Add test. (#11468)
------------------------------------------
[...truncated 64.52 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, chardet, urllib3, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587504051.855224136","description":"Error received from peer ipv4:127.0.0.1:45691","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0421212052-e37b13e3_5ae1302b-b2c1-4b01-8b7a-2c523baf37bf failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 18s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/kdcedvbhbplfy
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1263
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1263/display/redirect?page=changes>
Changes:
[boyuanz] Fix SplittableParDoNaiveBounded DoFnInvoker.
------------------------------------------
[...truncated 64.23 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, certifi, urllib3, idna, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587502736.664064252","description":"Error received from peer ipv4:127.0.0.1:39201","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0421205856-8139db69_37e0ae61-bf55-4b17-bd37-2174467b28ac failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 56s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/63froq2yyojre
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1262
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1262/display/redirect?page=changes>
Changes:
[pabloem] [BEAM-9608] BundleManagers use BundleContextManager for configuration
[pabloem] [BEAM-9639] Saving side inputs after producer execution, not before
[pabloem] [BEAM-9639] Separate Stage and Bundle execution. Improve typing
[pabloem] [BEAM-9639][BEAM-9608] Addressing review comments.
------------------------------------------
[...truncated 63.84 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, certifi, chardet, urllib3, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587498881.327597302","description":"Error received from peer ipv4:127.0.0.1:34037","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0421195441-5328ac87_df15dd8b-dd12-4acd-9841-acf70c5f7e6e failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
>>> FAILURE
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 8s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/fpgb3sxdhj3ri
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1261
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1261/display/redirect?page=changes>
Changes:
[github] [BEAM-9764] multi threading & optional pulling (#11428)
------------------------------------------
[...truncated 63.79 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, certifi, urllib3, chardet, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587496411.120440322","description":"Error received from peer ipv4:127.0.0.1:38747","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0421191331-9499726b_38a8c8f9-9447-4df9-b468-d60116854b3d failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 47s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/hv26e5w56zcxc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1260
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1260/display/redirect?page=changes>
Changes:
[ankurgoenka] [BEAM-9791] Add precommit for dataflow runner v2
[apilloud] [BEAM-9662][BEAM-9663] Reject Null Limit and Offset
[github] Merge pull request #11466 from [BEAM-9787] Clear error message on UW +
------------------------------------------
[...truncated 63.88 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, idna, chardet, certifi, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587494300.944225478","description":"Error received from peer ipv4:127.0.0.1:41641","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0421183821-b9167a3c_d7ffab15-e3fb-43cb-bb99-cb46c7323e3c failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 16s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/k2g3lspluofba
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1259
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1259/display/redirect>
Changes:
------------------------------------------
[...truncated 64.01 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, idna, chardet, urllib3, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587492634.734636947","description":"Error received from peer ipv4:127.0.0.1:34755","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0421181035-9029e7c3_f6e5b679-16ee-4de7-958e-f6eb49364c87 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 19s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/cc5hoh2heygg4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1258
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1258/display/redirect?page=changes>
Changes:
[github] Revert "[BEAM-8131] Provide Kubernetes setup for Prometheus (#9482)"
------------------------------------------
[...truncated 64.49 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, idna, chardet, certifi, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587488045.769251457","description":"Error received from peer ipv4:127.0.0.1:35951","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0421165406-9f17e5a5_e31c1fff-6ade-44f2-ba11-555e3f509c76 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 31s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/tuiefvvvc5mzm
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1257
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1257/display/redirect>
Changes:
------------------------------------------
[...truncated 64.47 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, certifi, chardet, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587470934.244358847","description":"Error received from peer ipv4:127.0.0.1:41033","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0421120854-fd3136bf_c2609d0b-4585-4bca-8faa-70eef0269244 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 35s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/y4tkktqmpnnlw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1256
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1256/display/redirect?page=changes>
Changes:
[iemejia] [BEAM-9285] Add Postcommit ValidatesRunner CI Job for Flink on Java 11
------------------------------------------
[...truncated 63.86 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, idna, urllib3, certifi, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587457986.035942011","description":"Error received from peer ipv4:127.0.0.1:36665","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0421083306-148271a2_c3735952-0c6b-46a2-9b7c-407fee43c599 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 56s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/g3fnkywbm4sw2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1255
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1255/display/redirect>
Changes:
------------------------------------------
[...truncated 64.24 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, chardet, idna, certifi, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587449578.041151902","description":"Error received from peer ipv4:127.0.0.1:40323","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0421061258-58f352e4_f465bddb-8c1e-4bde-b406-f72bd2cb1464 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 5s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/xkruiqihaiuru
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1254
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1254/display/redirect?page=changes>
Changes:
[robertwb] [BEAM-6960] More sophisticated job server starting and stopping.
[robertwb] Abort pipeline if docker containers die unexpectedly.
[robertwb] [BEAM-6960] Go ULR validates runner tests.
[robertwb] lint
------------------------------------------
[...truncated 64.04 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, chardet, idna, urllib3, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587433487.006742418","description":"Error received from peer ipv4:127.0.0.1:36211","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0421014447-6c3a1c7c_b8ba3a32-f3b6-4c76-945e-4be180a14166 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 0s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/xpjp7lgbdvbci
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1253
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1253/display/redirect?page=changes>
Changes:
[valentyn] Renames run_test -> run_scenario so that nose test runner does not think
------------------------------------------
[...truncated 64.25 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, idna, chardet, certifi, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587430397.053263218","description":"Error received from peer ipv4:127.0.0.1:35825","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0421005317-b83cdf44_702e8077-9644-44bc-a965-1d3bc6849509 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 10s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/m6dffffdxjnmg
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1252
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1252/display/redirect>
Changes:
------------------------------------------
[...truncated 64.30 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, certifi, chardet, urllib3, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587428170.997501785","description":"Error received from peer ipv4:127.0.0.1:35433","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0421001611-6475aa49_03f17166-bd24-4d30-a7ef-c5a9af74e7fd failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 12s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/jx5vyp2otdlio
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1251
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1251/display/redirect?page=changes>
Changes:
[iemejia] [BEAM-9569] Fix BeamSqlPojoExample logRecords method to not require Row
------------------------------------------
[...truncated 63.86 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, chardet, certifi, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587423344.030495216","description":"Error received from peer ipv4:127.0.0.1:34751","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0420225544-9deeec5f_3e91a12d-47f3-4ba0-8d70-42791f5b5aae failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 39s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/cjx7qgijycgcu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1250
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1250/display/redirect?page=changes>
Changes:
[kawaigin] add display:block to datatable so that the columns can have various
------------------------------------------
[...truncated 64.24 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, certifi, urllib3, chardet, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587420265.407672877","description":"Error received from peer ipv4:127.0.0.1:42089","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0420220425-34838b95_a5983ff6-626b-4d4e-81d5-e6d4e394a3b7 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 19s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/7obmvwy5mfivy
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1249
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1249/display/redirect?page=changes>
Changes:
[github] [BEAM-9496] Add to_dataframe and to_pcollection APIs. (#11264)
------------------------------------------
[...truncated 64.25 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, certifi, chardet, urllib3, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587418671.574793664","description":"Error received from peer ipv4:127.0.0.1:33075","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0420213751-d62dd49a_f8f001a3-dfa8-48c2-9788-9b5135746453 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 59s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/kwdmkldxkogyq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1248
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1248/display/redirect?page=changes>
Changes:
[github] [BEAM-9775] Add Go support for SDF StandardRequirements. (#11443)
------------------------------------------
[...truncated 64.25 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, chardet, urllib3, certifi, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587417035.937854971","description":"Error received from peer ipv4:127.0.0.1:37965","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0420211036-2864d880_c9fa2dc2-a96c-41ee-93c4-6fa078196f42 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
>>> FAILURE
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 13s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/s6kobd3urbj66
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1247
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1247/display/redirect?page=changes>
Changes:
[github] [BEAM-9788] Add RetainDockerContainers flag support to Go SDK. (#11467)
------------------------------------------
[...truncated 64.02 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, idna, certifi, urllib3, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587415351.391411018","description":"Error received from peer ipv4:127.0.0.1:46865","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0420204231-8b154211_15a43774-39f0-4668-b76e-2a8e8363cb29 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 21s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/hlozbt6zuyexu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1246
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1246/display/redirect?page=changes>
Changes:
[ehudm] [BEAM-6860] Fix iobase non-global windows bug
------------------------------------------
[...truncated 63.83 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, chardet, urllib3, certifi, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587413955.929221171","description":"Error received from peer ipv4:127.0.0.1:33141","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0420201916-eb67df38_454e066a-c542-4024-b5a6-d8df17585b20 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 43s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/a37tlkv4ynhvq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1245
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1245/display/redirect?page=changes>
Changes:
[skoonce] [BEAM-9773]: Update Dataflow Debug Capture to use Google API client
[huangry] Fix user distribution metric test by using less restricting value.
[huangry] Apply yapf fix.
[skoonce] Remove all Dataflow google-api-client-jackson(1) dependencies
[huangry] Replace all explicit validation.
[skoonce] Missed cleanup
------------------------------------------
[...truncated 64.02 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, idna, urllib3, chardet, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587412695.107247734","description":"Error received from peer ipv4:127.0.0.1:45211","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0420195815-f6676a77_df304b53-7ce4-4b12-bb41-1391b918ed78 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 30s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/2j4q63g3lq5ym
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1244
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1244/display/redirect?page=changes>
Changes:
[github] Add release date for 2.20.0
------------------------------------------
[...truncated 64.28 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, chardet, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587409063.682605775","description":"Error received from peer ipv4:127.0.0.1:41561","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0420185743-eb393880_7366556d-a1ac-4df7-a58a-e6ee84d37928 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 4s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/fegvd267rqdhq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1243
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1243/display/redirect>
Changes:
------------------------------------------
[...truncated 64.00 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, idna, chardet, certifi, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587407088.070387761","description":"Error received from peer ipv4:127.0.0.1:46389","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0420182448-5a387493_9e187f2a-8e8c-432a-92a2-de28e9a34cd8 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 34s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/6seldlepbld4q
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1242
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1242/display/redirect?page=changes>
Changes:
[github] [BEAM-9692]: Make CombineValues portable (#11335)
------------------------------------------
[...truncated 64.25 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, certifi, idna, urllib3, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587404688.553606393","description":"Error received from peer ipv4:127.0.0.1:39855","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0420174448-3bf330aa_cacfed1a-5355-41ff-874d-c24abae73a48 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 40s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/vrc23dp4q6kk4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1241
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1241/display/redirect?page=changes>
Changes:
[michal.walenia] [BEAM-9147] Add @Experimental annotation to VideoIntelligence related
[michal.walenia] Add @Experimental to the package-info as well
------------------------------------------
[...truncated 63.80 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, certifi, idna, urllib3, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587401025.654943211","description":"Error received from peer ipv4:127.0.0.1:41345","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0420164345-5dd2f6d5_6b44dda1-b9c2-448b-b6bc-367e0c3c601d failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 54s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/zgzskca2zme3c
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1240
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1240/display/redirect>
Changes:
------------------------------------------
[...truncated 63.98 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, idna, chardet, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587384546.613690612","description":"Error received from peer ipv4:127.0.0.1:45745","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0420120906-5a93f779_3d5ef8e1-b14a-434f-9c87-258dceb16712 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 48s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/zngbwheb3hfci
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1239
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1239/display/redirect>
Changes:
------------------------------------------
[...truncated 64.46 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, idna, certifi, urllib3, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587362931.269135758","description":"Error received from peer ipv4:127.0.0.1:39741","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0420060851-15b05743_39cb642b-ea52-4fee-8788-1c87b15fe2b3 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
>>> FAILURE
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 30s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/6yvbiwcjvwune
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1238
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1238/display/redirect>
Changes:
------------------------------------------
[...truncated 64.00 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Downloading numpy-1.18.3-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, chardet, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.3 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587341339.135879445","description":"Error received from peer ipv4:127.0.0.1:42071","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0420000859-f988d919_35ba722e-5c94-4348-80a7-f45050da9ee4 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 37s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/oiwao7mn24p6k
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1237
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1237/display/redirect>
Changes:
------------------------------------------
[...truncated 63.78 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, idna, chardet, urllib3, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587319731.642428766","description":"Error received from peer ipv4:127.0.0.1:46171","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0419180851-36eb9a67_a1549fd0-e1d8-474a-b2ac-a3931ae0ce13 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 33s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ux6u6kmqhyiho
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1236
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1236/display/redirect>
Changes:
------------------------------------------
[...truncated 64.24 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, idna, certifi, chardet, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587298238.377425763","description":"Error received from peer ipv4:127.0.0.1:44673","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0419121038-cf211579_38ebb8c0-127b-4a1a-934c-f6d2a67e84d2 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 18s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/g3fhzgcnc2jzs
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1235
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1235/display/redirect>
Changes:
------------------------------------------
[...truncated 64.24 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, certifi, idna, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587276528.916333209","description":"Error received from peer ipv4:127.0.0.1:37021","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0419060849-76a5b6e3_23f69c31-38c3-495e-ba9e-18463d98abdc failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 30s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/vskz6pqredm4y
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1234
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1234/display/redirect>
Changes:
------------------------------------------
[...truncated 64.24 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, certifi, idna, chardet, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587254947.197629375","description":"Error received from peer ipv4:127.0.0.1:44059","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0419000907-e21d363c_85211308-ddad-4e0a-9276-0adbf778785c failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 47s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ig3lvclqs4f2u
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1233
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1233/display/redirect>
Changes:
------------------------------------------
[...truncated 63.98 KB...]
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, chardet, idna, urllib3, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:root:Waiting for grpc channel to be ready at localhost:51133.
WARNING:root:Waiting for grpc channel to be ready at localhost:51133.
WARNING:root:Waiting for grpc channel to be ready at localhost:51133.
WARNING:root:Waiting for grpc channel to be ready at localhost:51133.
WARNING:root:Waiting for grpc channel to be ready at localhost:51133.
WARNING:root:Waiting for grpc channel to be ready at localhost:51133.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587233462.432222121","description":"Error received from peer ipv4:127.0.0.1:35963","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0418181103-c961f83a_25bf4a04-a8e4-49a3-a6c2-fd9af7849f41 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 47s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/2s3vpht3kncy6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1232
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1232/display/redirect>
Changes:
------------------------------------------
[...truncated 64.47 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, chardet, urllib3, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587211747.936577715","description":"Error received from peer ipv4:127.0.0.1:33419","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0418120908-3f8ed54b_c88f03a9-c0dd-447d-9bd6-587e6ebcfcfd failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 48s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/dui545w2tn3am
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1231
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1231/display/redirect>
Changes:
------------------------------------------
[...truncated 64.21 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, chardet, idna, certifi, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587190154.206409867","description":"Error received from peer ipv4:127.0.0.1:37627","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0418060914-7824eee8_18d5e4f1-f842-4a34-8ce7-b004b02ad709 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 54s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ohj6n6uk5hpsi
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1230
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1230/display/redirect?page=changes>
Changes:
[github] [BEAM-8872] Support split at fraction for OffsetRangeTracker (#11418)
------------------------------------------
[...truncated 64.69 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, urllib3, chardet, certifi, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587181793.175928910","description":"Error received from peer ipv4:127.0.0.1:42513","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0418034953-390b4058_b6ebae50-3f27-48d5-b855-9ce28d2b3a08 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 35s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/rlzwpgbn7js4o
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1229
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1229/display/redirect?page=changes>
Changes:
[github] [BEAM-3836] Enable dynamic splitting/checkpointing within the Java SDK
------------------------------------------
[...truncated 64.47 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, idna, certifi, chardet, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587179697.477820981","description":"Error received from peer ipv4:127.0.0.1:42549","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0418031457-e086298e_c1ee1c15-96f7-4f90-b168-b5501404832d failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 29s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/5w4i4lhvxzmrq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1228
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1228/display/redirect?page=changes>
Changes:
[robertwb] [BEAM-9577] Small fixes to portable runner staging.
------------------------------------------
[...truncated 64.03 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, urllib3, chardet, idna, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587168713.612553161","description":"Error received from peer ipv4:127.0.0.1:40079","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0418001153-9fe491d7_cabb24a8-5824-4989-8d56-6c72a578c76b failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 58s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/dv5zk52c6coq2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1227
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1227/display/redirect?page=changes>
Changes:
[github] Updates and clarifications on type safety. (#11449)
------------------------------------------
[...truncated 64.04 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, chardet, idna, urllib3, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587163361.391422275","description":"Error received from peer ipv4:127.0.0.1:33919","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0417224241-fc803ec6_b2a05d70-0db6-45bc-bb41-981f95c856d2 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 14s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/gabsvugicudvi
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1226
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1226/display/redirect?page=changes>
Changes:
[APryor] Correct a typo in documentation
------------------------------------------
[...truncated 64.29 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, certifi, idna, urllib3, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587162413.443610164","description":"Error received from peer ipv4:127.0.0.1:37673","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0417222653-a56333eb_8ebbb2d9-2924-4ebe-85cd-b75ecaaf883a failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 53s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/w2glcc6eh2k2s
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1225
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1225/display/redirect?page=changes>
Changes:
[github] [BEAM-9743] Fix TFRecordCodec to try harder to read/write (#11397)
------------------------------------------
[...truncated 530.42 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, idna, chardet, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587161420.287560384","description":"Error received from peer ipv4:127.0.0.1:41635","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0417221020-e53cf59c_37c70620-66fd-4d7e-a44c-20f119c6c9d8 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 42s
65 actionable tasks: 60 executed, 4 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ezzuxv7onjrka
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1224
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1224/display/redirect?page=changes>
Changes:
[robertwb] [BEAM-9768] Gradle command for Python ULR.
[github] [BEAM-8646] Fix external environment on OS X as well. (#11431)
------------------------------------------
[...truncated 64.30 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, chardet, certifi, idna, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587151766.953424853","description":"Error received from peer ipv4:127.0.0.1:32975","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0417192927-ee65f406_bef8d7d0-a1cd-45d8-80f7-892208ced1db failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 44s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/6bluq4fbwbhtg
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1223
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1223/display/redirect?page=changes>
Changes:
[github] [BEAM-9650] Cleanup documentation on side inputs patterns (#11415)
------------------------------------------
[...truncated 64.06 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, certifi, idna, chardet, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587149137.227918397","description":"Error received from peer ipv4:127.0.0.1:35807","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0417184537-72331a58_98527b34-d89b-48fa-aa26-dcb903a46416 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 56s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/snbjlawm5pduu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1222
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1222/display/redirect>
Changes:
------------------------------------------
[...truncated 64.23 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, urllib3, chardet, certifi, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587147103.263107953","description":"Error received from peer ipv4:127.0.0.1:46827","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0417181143-fd5834fa_028daac2-67e1-4601-ba8a-b6bc81d0e233 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 10s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/5xzoavk37kdgu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1221
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1221/display/redirect?page=changes>
Changes:
[ehudm] [BEAM-9737] Don't use docker create option -u
------------------------------------------
[...truncated 64.22 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, idna, certifi, urllib3, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587146246.775482897","description":"Error received from peer ipv4:127.0.0.1:37215","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0417175727-56c9fab0_d606cf67-ce6e-4b32-ae5b-c2ddc570f5b2 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 30s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/uh6kn7ga7cmco
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1220
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1220/display/redirect?page=changes>
Changes:
[github] [BEAM-9674] Don't specify selected fields when fetching BigQuery table
------------------------------------------
[...truncated 63.81 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, certifi, idna, urllib3, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587135011.883429928","description":"Error received from peer ipv4:127.0.0.1:45841","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0417145012-11bfd79a_8ff019f7-ba04-4323-a46b-12a870174bb8 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 53s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/6n3wtytogzoxi
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1219
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1219/display/redirect>
Changes:
------------------------------------------
[...truncated 64.25 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, chardet, idna, certifi, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587125387.840029946","description":"Error received from peer ipv4:127.0.0.1:42175","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0417120948-c8326d73_91d3bd00-fd27-4d80-bf00-89ea809fe472 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 28s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/gglhyhivrm52c
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1218
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1218/display/redirect>
Changes:
------------------------------------------
[...truncated 64.48 KB...]
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, certifi, urllib3, idna, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:root:Waiting for grpc channel to be ready at localhost:42367.
WARNING:root:Waiting for grpc channel to be ready at localhost:42367.
WARNING:root:Waiting for grpc channel to be ready at localhost:42367.
WARNING:root:Waiting for grpc channel to be ready at localhost:42367.
WARNING:root:Waiting for grpc channel to be ready at localhost:42367.
WARNING:root:Waiting for grpc channel to be ready at localhost:42367.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587103898.849408315","description":"Error received from peer ipv4:127.0.0.1:37625","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0417061140-950e98cd_7f1bda09-10d2-46bb-949e-0c1522fb29e2 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
>>> FAILURE
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 11m 24s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/3iqe6bon4xoxu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1217
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1217/display/redirect?page=changes>
Changes:
[huangry] Include user distritribution into metric-dedicated validate runner test.
[huangry] update inline comment.
[huangry] To allow only assert on committed metrics, bypassing the attempted ones.
[huangry] Use distribution matcher for testing.
[github] [BEAM-5605, BEAM-2939] Add support for FnApiDoFnRunner to handle split
------------------------------------------
[...truncated 64.28 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, certifi, chardet, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587085738.657011029","description":"Error received from peer ipv4:127.0.0.1:46379","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0417010858-3c6455b3_32a538a3-9a8c-44ba-890a-5ff9eceeaccf failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 8s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/en2zitsei6rp4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1216
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1216/display/redirect>
Changes:
------------------------------------------
[...truncated 63.77 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, idna, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587082368.006491523","description":"Error received from peer ipv4:127.0.0.1:41033","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0417001248-e72ccf4a_81c79990-2b9c-4ff0-a0ba-514b423c8ca6 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 4s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/x2j4vhew54oek
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1215
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1215/display/redirect?page=changes>
Changes:
[github] [BEAM-5379] Ignore go.sum files in RAT checks
------------------------------------------
[...truncated 63.80 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, chardet, certifi, idna, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587077811.855666201","description":"Error received from peer ipv4:127.0.0.1:40987","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416225652-1d594c2c_5a2705d5-8395-496e-b1b5-6c425dbc2c46 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 36s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/kxfjsbw3alrhk
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1214
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1214/display/redirect?page=changes>
Changes:
[github] [BEAM-9648]: DirectRunner should return null on timeout (#11275)
------------------------------------------
[...truncated 64.02 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, idna, urllib3, chardet, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587069037.548201712","description":"Error received from peer ipv4:127.0.0.1:34177","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416203037-50dda51b_0e5dfb32-3476-480f-87c1-9ab3a890fbfc failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 58s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/ufpy2zcycjeki
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1213
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1213/display/redirect?page=changes>
Changes:
[github] Merge pull request #11436: Document Beam Schemas (forked PR)
------------------------------------------
[...truncated 64.25 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, certifi, urllib3, idna, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587066584.841114072","description":"Error received from peer ipv4:127.0.0.1:34977","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416194945-4a38273f_ae9c4a21-356f-4fec-9ca3-80e931e1f0c7 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
>>> FAILURE
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 6s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/7chjelhpdqi7a
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1212
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1212/display/redirect?page=changes>
Changes:
[pabloem] [BEAM-9769] Ensuring JSON is the default export format for BQ sink
------------------------------------------
[...truncated 63.84 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, chardet, idna, urllib3, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587064819.668471557","description":"Error received from peer ipv4:127.0.0.1:45829","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416192019-63492db5_af40be28-e67d-4948-8f3e-baee83ede348 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 39s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/uuyzjuxeease2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1211
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1211/display/redirect>
Changes:
------------------------------------------
[...truncated 64.01 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, chardet, idna, urllib3, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587062289.869859952","description":"Error received from peer ipv4:127.0.0.1:38443","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416183810-26fd0f29_2fbefe01-630a-4c00-9374-8c0370b724fa failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 58s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/y42xl23jhu3hg
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1210
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1210/display/redirect?page=changes>
Changes:
[github] [BEAM-9678] Create Go SDK introduction kata (#11340)
------------------------------------------
[...truncated 63.99 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, idna, certifi, chardet, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587060771.197016743","description":"Error received from peer ipv4:127.0.0.1:40189","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416181251-72cec391_628ed003-d887-4754-803f-9e8552a17a68 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 28s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/tm2bneqfw33mu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1209
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1209/display/redirect?page=changes>
Changes:
[kcweaver] [BEAM-9765] Exclude module-info.class from vendored Calcite.
------------------------------------------
[...truncated 63.77 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Downloading urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, idna, certifi, urllib3, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587055608.950537826","description":"Error received from peer ipv4:127.0.0.1:34969","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416164649-f50322b8_86f88523-2501-4c2e-b4a1-d4cb130677a9 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 50s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/t7nczvvvy4gio
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1208
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1208/display/redirect?page=changes>
Changes:
[echauchot] Add GBK load tests script for spark structured streaming runner
[echauchot] Add CoGBK load tests script for spark structured streaming runner
[echauchot] Add Combine load tests script for spark structured streaming runner
[echauchot] Add Pardo load tests script for spark structured streaming runner
------------------------------------------
[...truncated 63.80 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, certifi, idna, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587041936.451092494","description":"Error received from peer ipv4:127.0.0.1:38957","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416125856-b99d8043_271c5adb-28ba-4c6f-bd02-2b9faad0233d failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 11s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/zstokgdy6gojs
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1207
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1207/display/redirect>
Changes:
------------------------------------------
[...truncated 63.79 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, idna, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587038958.544179328","description":"Error received from peer ipv4:127.0.0.1:41687","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416120918-bc13fe5b_e530106a-d7a0-4945-9a99-309b045bf87e failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 59s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/hstlp3qc3cude
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1206
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1206/display/redirect?page=changes>
Changes:
[kamil.wasilewski] Remove outdated doc for ReadFromBigQuery transform
------------------------------------------
[...truncated 64.24 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, chardet, certifi, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587028507.288660108","description":"Error received from peer ipv4:127.0.0.1:42415","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416091507-30c8e923_1acbd053-d8f0-4875-b32f-45c767d39d6a failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 59s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/jcddgt3exgvow
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1205
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1205/display/redirect?page=changes>
Changes:
[kamil.wasilewski] [BEAM-8671] Migrate Load Tests to Python 3.7
------------------------------------------
[...truncated 64.02 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, urllib3, certifi, idna, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587027953.812938157","description":"Error received from peer ipv4:127.0.0.1:42601","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416090554-fb3b6c8b_1e03a4cd-8612-489d-a0d0-4128bf315081 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 19s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/j2rmkpumde2q4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1204
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1204/display/redirect?page=changes>
Changes:
[jkw] Fix typo
------------------------------------------
[...truncated 64.02 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, idna, urllib3, chardet, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587022912.877923259","description":"Error received from peer ipv4:127.0.0.1:33893","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416074153-48ab658b_ba1b4dfb-7739-48a1-86fd-b65db61e1fcc failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 31s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/axthwh426pjho
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1203
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1203/display/redirect>
Changes:
------------------------------------------
[...truncated 64.66 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, chardet, urllib3, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587017344.853238698","description":"Error received from peer ipv4:127.0.0.1:37247","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416060905-47e0fbd9_84b788d2-5c3f-4907-afc1-741b5106fd39 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 46s
65 actionable tasks: 51 executed, 13 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/y4bpqzgqqu64w
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1202
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1202/display/redirect?page=changes>
Changes:
[github] Merge pull request #11151 from [BEAM-9468] Hl7v2 io
------------------------------------------
[...truncated 63.78 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, certifi, chardet, idna, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1587004321.092957414","description":"Error received from peer ipv4:127.0.0.1:46475","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416023201-9828d39c_2f8477a3-322a-4430-b320-1559400f508c failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 15s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/kboahhwwu6qvc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1201
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1201/display/redirect>
Changes:
------------------------------------------
[...truncated 63.74 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, urllib3, chardet, idna, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1586996599.950792339","description":"Error received from peer ipv4:127.0.0.1:38441","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0416002320-62551b19_d910090e-5a9a-4ecc-bcd0-b16856db4460 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 5s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/jghhxkrblcgva
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1200
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1200/display/redirect?page=changes>
Changes:
[amaliujia] add 2.20.0 blog post
[amaliujia] add a known issue
[amaliujia] fixup! update 2.20.0 date
[github] Add new release 2.20.0 to beam website (#11285)
------------------------------------------
[...truncated 63.79 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, urllib3, chardet, idna, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1586991695.596834843","description":"Error received from peer ipv4:127.0.0.1:40355","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0415230135-9ec029c1_cacdf1b2-f5e1-4e9b-89a9-676f746bb63c failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 45s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/gdhwvvogzyfbm
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1199
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1199/display/redirect?page=changes>
Changes:
[thw] Fix py37-lint
[thw] Maven compatible publish repository authentication via settings.xml
[github] [BEAM-9729, BEAM-8486] Runner-side bundle registration cleanup. (#11358)
------------------------------------------
[...truncated 64.92 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, idna, certifi, urllib3, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1586986764.826519230","description":"Error received from peer ipv4:127.0.0.1:34283","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0415213925-f3acf7a4_7802a821-23ef-436d-b922-4d892a1ee06d failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 45s
65 actionable tasks: 52 executed, 12 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/v3lftf36eidj4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1198
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1198/display/redirect?page=changes>
Changes:
[ehudm] [BEAM-8466] Make strip_iterable more strict
[pabloem] Fixing type names for BQ Avro Tools
[ehudm] [BEAM-9119] Disable flaky test
------------------------------------------
[...truncated 63.82 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, idna, chardet, certifi, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1586978839.653945421","description":"Error received from peer ipv4:127.0.0.1:33963","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0415192719-2e30fcfc_e28fbc66-759e-42c9-852f-87c2153eceec failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 55s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/wly6k3o5lrjaw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1197
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1197/display/redirect?page=changes>
Changes:
[kcweaver] [BEAM-9756] Nexmark: only use --region in Dataflow.
[github] [BEAM-8889] add gRPC suport in GCS connector (behind an
------------------------------------------
[...truncated 63.85 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, docopt, certifi, urllib3, idna, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1586976129.308714139","description":"Error received from peer ipv4:127.0.0.1:43991","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0415184209-c1283903_44c81f32-a4f5-46a1-b2b1-c29ac971fc93 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 7m 43s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/jsrbkkpbm2vl4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1196
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1196/display/redirect?page=changes>
Changes:
[pabloem] Removing underscore from _ReadFromBigQuery to make it external. It
------------------------------------------
[...truncated 64.30 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, urllib3, certifi, idna, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1586972814.557030217","description":"Error received from peer ipv4:127.0.0.1:35717","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0415174654-19a8552_16c8fa18-f21b-4a50-ba78-cfcd1414ac46 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 8m 17s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/pze7pyzstct6i
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1195
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1195/display/redirect?page=changes>
Changes:
[lcwik] [BEAM-9577] Fix test to create urls from paths which are compatible with
------------------------------------------
[...truncated 64.69 KB...]
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.22,>=0.21.4
Using cached fastavro-0.21.24-cp37-cp37m-manylinux1_x86_64.whl (1.2 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.28.1-cp37-cp37m-manylinux2010_x86_64.whl (2.8 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Processing /home/jenkins/.cache/pip/wheels/0d/e7/b6/0dd30343ceca921cfbd91f355041bd9c69e0f40b49f25b7b8a/httplib2-0.12.0-py3-none-any.whl
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.2-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.11.3-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2019.3-py2.py3-none-any.whl (509 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.17.0,>=0.15.1
Using cached pyarrow-0.16.0-cp37-cp37m-manylinux2014_x86_64.whl (63.1 MB)
Collecting six>=1.5.2
Using cached six-1.14.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.22.0.dev0) (46.1.3)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.8-py2.py3-none-any.whl (125 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, fastavro, future, six, grpcio, chardet, urllib3, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, pytz, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.21.24 future-0.18.2 grpcio-1.28.1 hdfs-2.5.8 httplib2-0.12.0 idna-2.9 mock-2.0.0 numpy-1.18.2 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.11.3 pyarrow-0.16.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2019.3 requests-2.23.0 rsa-4.0 six-1.14.0 typing-extensions-3.7.4.2 urllib3-1.25.8
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:grpc._channel:Exception iterating requests!
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 195, in consume_request_iterator
request = next(request_iterator)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 316, in __next__
raise self._queue.get()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 192, in stage
return self._stage_via_portable_service(channel, staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 210, in _stage_via_portable_service
staging_session_token)
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/artifact_service.py",> line 506, in offer_artifacts
for request in requests:
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 416, in __next__
return self._next()
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/lib/python3.7/site-packages/grpc/_channel.py",> line 706, in _next
raise self
grpc._channel._MultiThreadedRendezvous: <_MultiThreadedRendezvous of RPC that terminated with:
status = StatusCode.UNIMPLEMENTED
details = "Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService"
debug_error_string = "{"created":"@1586965856.948024893","description":"Error received from peer ipv4:127.0.0.1:46731","file":"src/core/lib/surface/call.cc","file_line":1056,"grpc_message":"Method not found: org.apache.beam.model.job_management.v1.ArtifactStagingService/ReverseArtifactRetrievalService","grpc_status":12}"
>
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 571, in wait_until_finish
(self._job_id, self._state, self._last_error_message()))
RuntimeError: Pipeline BeamApp-jenkins-0415155057-f6111aac_81b476e9-c781-46d9-9014-16cf4be064f7 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
>>> FAILURE
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 160
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 28s
65 actionable tasks: 50 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/xfxh5aojmu764
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org