You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@beam.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2020/05/28 18:15:56 UTC
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1451
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1451/display/redirect?page=changes>
Changes:
[kmjung] Reject fractional values outside of (0.0, 1.0)
[aramaswamis] [BEAM-10067] Minify website assets with --minify flag
[altay] Add a bare bones powered by page.
[kcweaver] [BEAM-10106] Script the deployment of artifacts to pypi
[kcweaver] Add pypi script to release guide.
[kcweaver] Improve log messages in release scripts.
[kcweaver] Install twine in virtual env.
[github] Avoid re-encoding value state cells each commit (#11823)
------------------------------------------
[...truncated 64.41 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.0)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, certifi, urllib3, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0528181551-1b690b64_9aaf6331-beb7-4af1-8efe-804da908c875 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 15s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/lyn76bk2cguf2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1476
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1476/display/redirect?page=changes>
Changes:
[github] Always use python3 for license script virtualenv (#11900)
[github] fix hub existence test (#11903)
[github] Merge pull request #11896 from [BEAM-8841] Make bigquery_avro_tools_test
[kcweaver] [BEAM-10187] Push release candidate tag to Github.
[github] [BEAM-10166] Context improvements for exec errors. (#11881)
------------------------------------------
[...truncated 64.61 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, chardet, idna, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0604001038-22b3f232_48b220fb-bda0-4a41-873d-72ee8e8de574 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 17s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/7gmtd6qiziozc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1475
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1475/display/redirect?page=changes>
Changes:
[david.moravek] [BEAM-10164] Flink Batch Runner: Memory efficient combine implementation
[aromanenko.dev] [BEAM-10096] Fix Spark runners numbering
[iemejia] [BEAM-9948] Fix mascot invalid directory link
[github] [BEAM-9646] Add Google Cloud vision integration transform (#11331)
------------------------------------------
[...truncated 64.17 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, chardet, idna, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0603181034-3467112a_7897e7da-5bad-4721-89a3-b1362dc5dd60 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 11s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/dqcbi4w2ht37i
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1474
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1474/display/redirect?page=changes>
Changes:
[piotr.szuberski] [BEAM-7774] Move duplicated code from py2 and py37 suites to
[piotr.szuberski] [BEAM-7774] Remove Perfkit Benchmarking tool from python wordcount
[piotr.szuberski] [BEAM-7774] Add python wordcount performance test grafana dashboard
[kcweaver] [BEAM-10132] Update Docker images in Flink runner page.
[aizhamal.nk] New directory for Beam mascot assets
[github] [BEAM-5863] Jenkins job to deploy Beam Metrics infrastructure
[github] Fix an issue with kubectl not being configured properly (#11906)
------------------------------------------
[...truncated 63.96 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, urllib3, chardet, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0603121442-52b47473_cc03790e-9af8-4456-9796-733bef3cc8f0 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 18s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/bijpdvcxikbda
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1473
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1473/display/redirect?page=changes>
Changes:
[github] Update fastavro_it_test.py
[github] Update wordcount_debugging.py
[github] Update wordcount_minimal.py
[github] fix spacing
[github] add example region
[github] lint
[kcweaver] [BEAM-10168] Add Github 'publish release' to release guide
[github] Add a GCThrashingPerPeriod option for MemoryMonitor (#11885)
[github] [BEAM-9946] | added new api in Partition Transform (#11682)
------------------------------------------
[...truncated 64.87 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, chardet, certifi, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0603061026-d9bbed75_f9d369f2-bbd7-4393-8c3a-2f1f889860f0 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 3s
67 actionable tasks: 52 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/xruaurnokk6zm
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1472
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1472/display/redirect?page=changes>
Changes:
[kcweaver] [BEAM-10167] Fix 2.21.0 downloads link in blog post
[iemejia] [BEAM-10173] Upgrade zstd-jni to version 1.4.5-2
[bhulette] Remove release note cleanup task
[github] [BEAM-4766] Migrate to use vendored protobuf for bytestring (#11891)
[github] [BEAM-10027] Support for Kotlin-based Beam Katas (#11761)
[github] [BEAM-10179] Clean-up URNJavaDoFn workaround in Go SDK. (#11899)
[github] [BEAM-10063] Create a WontImplementError for skipping features. (#11785)
[github] Remove conditional grpc imports. (#11878)
------------------------------------------
[...truncated 64.38 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, certifi, idna, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0603002203-bc434824_c2af53ba-9320-4e5d-b45b-b3b57f36c933 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 11m 25s
67 actionable tasks: 52 executed, 14 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/lxd64gy4ljtxw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1471
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1471/display/redirect?page=changes>
Changes:
[piotr.szuberski] [BEAM-10171] Add Kinesis, Spanner, Jdbc IOs to the portability roadmap
[david.moravek] [BEAM-8850,BEAM-8849] FlinkBatchTranslationContext respects input
[david.moravek] [BEAM-8848] Flink Batch Runner: optimize GBK translation for non-merging
[david.moravek] [BEAM-8848] Code review.
[klk] Update Jira priorities to the new Beam custom priority scheme
------------------------------------------
[...truncated 64.18 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, certifi, urllib3, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0602182941-ab7df916_e8124ef1-7be0-4917-8f54-aaac06ffefd0 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 14s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/zhdixm4qni56q
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1470
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1470/display/redirect?page=changes>
Changes:
[github] [BEAM-9723] Add DLP integration transforms (#11566)
------------------------------------------
[...truncated 64.18 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, certifi, chardet, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0602121005-e9282d4f_e82cfdd5-4b89-48f3-a369-cf66ee64bb5e failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 42s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/kiasgscv4npva
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1469
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1469/display/redirect?page=changes>
Changes:
[github] [BEAM-9825] | Implement Intersect,Union,Except transforms (#11610)
------------------------------------------
[...truncated 582.45 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, chardet, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0602061100-9ac5b9bc_a2eb0ccb-4546-4a4a-84d0-672d68464820 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 38s
67 actionable tasks: 61 executed, 5 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/nmfku7dgm6ffi
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1468
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1468/display/redirect?page=changes>
Changes:
[kcweaver] [BEAM-10154] Remove stray version number.
[github] Update programming-guide.md
[github] Updates CHANGES.md to include new I/Os for Beam 2.22.0. (#11879)
[github] [BEAM-10158] Use a shared unbounded thread pool within Beam Python.
------------------------------------------
[...truncated 64.20 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, urllib3, chardet, idna, certifi, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0602001030-a7d2e935_aac876db-709e-46d3-929a-0636783f9fd8 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 6s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/fn7dkjzlwiudc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1467
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1467/display/redirect?page=changes>
Changes:
[github] [BEAM-10162, BEAM-10163] Change PubSubIO and BigQueryIO job names;
[github] [BEAM-10051] Scope instruction data. (#11864)
[github] [BEAM-9679] Add Combine/Simple Function to Core Transforms (#11866)
------------------------------------------
[...truncated 64.43 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0601181000-f26b72a5_bd91fae9-3ae3-42b5-ad50-fbe35599cc05 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 40s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/tzf43sm7xvwdo
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1466
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1466/display/redirect?page=changes>
Changes:
[david.moravek] [BEAM-8608] Chain DoFns in Flink batch runner when possible.
[david.moravek] [BEAM-8608] Code review. Fix checkstyle.
[github] [BEAM-9421] Better documentation of output results from AnnotateText
[github] [BEAM-9421] Website section that describes getting predictions using AI
------------------------------------------
[...truncated 64.46 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, certifi, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0601121018-20e03fc9_02e873d2-057d-4347-9466-b93ed0e5c176 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 54s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/uwvcdtwwfkat4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1465
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1465/display/redirect>
Changes:
------------------------------------------
[...truncated 63.96 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, urllib3, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0601060946-fcee7624_4ccf5179-85d8-415b-9f8f-bdce96a8653c failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 26s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/obkn5ammda5ag
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1464
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1464/display/redirect>
Changes:
------------------------------------------
[...truncated 64.19 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, certifi, urllib3, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0601000942-1f3045d1_f3b494dd-9dc8-4b18-a49f-04d68cb0a7df failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 22s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/3dubxayrch3j2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1463
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1463/display/redirect>
Changes:
------------------------------------------
[...truncated 64.55 KB...]
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, urllib3, certifi, idna, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:root:Waiting for grpc channel to be ready at localhost:53187.
WARNING:root:Waiting for grpc channel to be ready at localhost:53187.
WARNING:root:Waiting for grpc channel to be ready at localhost:53187.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0531181307-7069d49d_d07c2274-aaae-4f66-83ea-c820875bf53a failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
>>> FAILURE
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 12m 50s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/frubrjq2udowe
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1462
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1462/display/redirect>
Changes:
------------------------------------------
[...truncated 64.19 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, idna, certifi, urllib3, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0531121000-76ae6448_8dd702b2-345a-4872-ac96-fa13aa9a4149 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 38s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/zc3ecaylayza6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1461
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1461/display/redirect>
Changes:
------------------------------------------
[...truncated 63.96 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, idna, urllib3, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0531060940-2eb5e68c_1fb3d074-47c7-484e-9d15-d5c19449c573 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
>>> FAILURE
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 20s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/v2lu7uvmscg2k
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1460
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1460/display/redirect>
Changes:
------------------------------------------
[...truncated 64.42 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, chardet, idna, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0531000938-63ef48d8_9dcca4d0-fed6-4680-8514-8c9320f6361e failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
>>> FAILURE
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 18s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/exmvpjltcjxja
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1459
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1459/display/redirect?page=changes>
Changes:
[github] Update programming-guide.md
------------------------------------------
[...truncated 64.19 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, idna, chardet, urllib3, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0530181051-5c28483a_e4290e71-d04f-49bb-b363-44b1c371a355 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 30s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/4chbkae6m6l46
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1458
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1458/display/redirect>
Changes:
------------------------------------------
[...truncated 64.22 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, chardet, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0530120947-59c8afb0_49af2229-f378-41b3-ba28-f60010aa4f27 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 27s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/znyvwqwnodnx2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1457
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1457/display/redirect>
Changes:
------------------------------------------
[...truncated 64.00 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, certifi, urllib3, idna, chardet, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0530060931-5cbd6b6e_079ba691-33ab-4d87-a182-236c6f2445e1 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 10s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/3dy74n6x5tgcw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1456
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1456/display/redirect?page=changes>
Changes:
[heejong] [BEAM-10125] adding cross-language KafkaIO integration test
[rohde.samuel] Moves apply_WriteToBigQuery to override
[github] [BEAM-10121] Python RowCoder doesn't support nested structs (#11841)
[rohde.samuel] Skip test if py=3.8 see BEAM-9754
[github] [BEAM-7746] Fix type errors and enable checks for
------------------------------------------
[...truncated 64.22 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, certifi, urllib3, idna, chardet, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0530001006-bbe9469f_a2159133-1b95-4008-af35-8b62c053cd0d failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 44s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/3vao7wuzs5qle
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1455
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1455/display/redirect?page=changes>
Changes:
[bhulette] remove as_cloud_object override
[sniemitz] [BEAM-9964] Move workerCacheMb to a user-visible place
[github] [BEAM-10078] uniquify Dataflow specific jars when staging (#11814)
[github] [BEAM-10107] Remove outdated instructions for website updates in releā¦
[github] [BEAM-9679] Add a CoGroupByKey lesson to the Core Transforms section
[github] [BEAM-10110] Propagate ids for custom coders. (#11832)
------------------------------------------
[...truncated 63.95 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, certifi, urllib3, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0529181334-692d5f_49b14aad-cf83-4341-bd2f-60654391d3c1 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 5s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/6tsnwr26ncgf2
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1454
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1454/display/redirect?page=changes>
Changes:
[aromanenko.dev] [BEAM-9916] Fixed some IO Urls on website
------------------------------------------
[...truncated 64.17 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, docopt, chardet, urllib3, certifi, idna, requests, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0529121215-c40fdf7c_07e2e8be-1979-4653-a9b1-b87d11f0eb32 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 11m 58s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/23zidlepjzweg
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1453
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1453/display/redirect>
Changes:
------------------------------------------
[...truncated 64.19 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, chardet, certifi, idna, urllib3, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0529060931-5fa37f0b_a9618688-1a98-4e18-a7aa-5fa7d4cf3872 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 9m 11s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/7xkqv7zbuknss
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #1452
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/1452/display/redirect?page=changes>
Changes:
[chamikaramj] Enables proto holders for testing runner API. Without this x-lang can be
[github] [BEAM-9679] Flatten Kata for Go (#11806)
[github] [BEAM-9220] Go Dataflow jobs to use runner v2 (#11207)
[robertwb] Update multi-language roadmap status.
[daniel.o.programmer] [BEAM-9935] Respect allowed split points and fraction in Go.
[github] [BEAM-8280] Blog post: Python typing changes (#11070)
[github] Update website/www/site/content/en/roadmap/connectors-multi-sdk.md
------------------------------------------
[...truncated 63.88 KB...]
# Verify container has already been built
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
apache/beam_python3.7_sdk:2.23.0.dev
# Set up Python environment
virtualenv -p python$PYTHON_VERSION $ENV_DIR
Using base prefix '/usr'
New python executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python3.7>
Also creating executable in <https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407/bin/python>
Installing setuptools, pip, wheel...
done.
Running virtualenv with interpreter /usr/bin/python3.7
. $ENV_DIR/bin/activate
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/-2047437407">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1-}"
if [ "x" != x ] ; then
PS1="${PS1-}"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) ${PS1-}"
fi
export PS1
fi
basename "$VIRTUAL_ENV"
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc || true
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Processing /home/jenkins/.cache/pip/wheels/dc/9a/e9/49e627353476cec8484343c4ab656f1e0d783ee77b9dde2d1f/crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Processing /home/jenkins/.cache/pip/wheels/a4/61/fd/c57e374e580aa78a45ed78d5859b3a44436af17e22ca53284f/dill-0.3.1.1-py3-none-any.whl
Collecting fastavro<0.24,>=0.21.4
Using cached fastavro-0.23.4-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
Processing /home/jenkins/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0/future-0.18.2-py3-none-any.whl
Collecting grpcio<2,>=1.12.1
Using cached grpcio-1.29.0-cp37-cp37m-manylinux2010_x86_64.whl (3.0 MB)
Processing /home/jenkins/.cache/pip/wheels/0a/7d/38/ea4eaf831518e6cd867b515b88919a9785eb66f11def5ab859/hdfs-2.5.8-py3-none-any.whl
Collecting httplib2<0.18.0,>=0.8
Using cached httplib2-0.17.4-py3-none-any.whl (95 kB)
Collecting mock<3.0.0,>=1.0.1
Using cached mock-2.0.0-py2.py3-none-any.whl (56 kB)
Collecting numpy<2,>=1.14.3
Using cached numpy-1.18.4-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.10.1-cp37-cp37m-manylinux2014_x86_64.whl (462 kB)
Processing /home/jenkins/.cache/pip/wheels/86/73/7a/3b3f76a2142176605ff38fbca574327962c71e25a43197a4c1/oauth2client-3.0.0-py3-none-any.whl
Collecting protobuf<4,>=3.5.0.post1
Using cached protobuf-3.12.2-cp37-cp37m-manylinux1_x86_64.whl (1.3 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.1-py2.py3-none-any.whl (19 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.1-py2.py3-none-any.whl (227 kB)
Collecting pytz>=2018.3
Using cached pytz-2020.1-py2.py3-none-any.whl (510 kB)
Collecting typing-extensions<3.8.0,>=3.7.0
Using cached typing_extensions-3.7.4.2-py3-none-any.whl (22 kB)
Processing /home/jenkins/.cache/pip/wheels/bc/49/5f/fdb5b9d85055c478213e0158ac122b596816149a02d82e0ab1/avro_python3-1.9.2.1-py3-none-any.whl
Collecting pyarrow<0.18.0,>=0.15.1
Using cached pyarrow-0.17.1-cp37-cp37m-manylinux2014_x86_64.whl (63.8 MB)
Collecting six>=1.5.2
Using cached six-1.15.0-py2.py3-none-any.whl (10 kB)
Collecting requests>=2.7.0
Using cached requests-2.23.0-py2.py3-none-any.whl (58 kB)
Processing /home/jenkins/.cache/pip/wheels/9b/04/dd/7daf4150b6d9b12949298737de9431a324d4b797ffd63f526e/docopt-0.6.2-py2.py3-none-any.whl
Collecting pbr>=0.11
Using cached pbr-5.4.5-py2.py3-none-any.whl (110 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.0-py2.py3-none-any.whl (38 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Requirement already satisfied: setuptools in ./build/gradleenv/-2047437407/lib/python3.7/site-packages (from protobuf<4,>=3.5.0.post1->apache-beam==2.23.0.dev0) (47.1.1)
Collecting pyparsing>=2.1.4
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
Using cached urllib3-1.25.9-py2.py3-none-any.whl (126 kB)
Collecting idna<3,>=2.5
Using cached idna-2.9-py2.py3-none-any.whl (58 kB)
Collecting chardet<4,>=3.0.2
Using cached chardet-3.0.4-py2.py3-none-any.whl (133 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2020.4.5.1-py2.py3-none-any.whl (157 kB)
Installing collected packages: crcmod, dill, pytz, fastavro, future, six, grpcio, urllib3, idna, chardet, certifi, requests, docopt, hdfs, httplib2, pbr, mock, numpy, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, protobuf, pyparsing, pydot, python-dateutil, typing-extensions, avro-python3, pyarrow, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.2.1 certifi-2020.4.5.1 chardet-3.0.4 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-0.23.4 future-0.18.2 grpcio-1.29.0 hdfs-2.5.8 httplib2-0.17.4 idna-2.9 mock-2.0.0 numpy-1.18.4 oauth2client-3.0.0 pbr-5.4.5 protobuf-3.12.2 pyarrow-0.17.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.1 pymongo-3.10.1 pyparsing-2.4.7 python-dateutil-2.8.1 pytz-2020.1 requests-2.23.0 rsa-4.0 six-1.15.0 typing-extensions-3.7.4.2 urllib3-1.25.9
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_config=$PYTHON_CONTAINER_IMAGE \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 583, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0529012517-6b357be8_2c00e003-bbe0-451e-82a4-386184dad9b0 failed in state FAILED: java.lang.IllegalArgumentException: Expected exactly one jar on sun.misc.Launcher$AppClassLoader@1b6d3586
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :runners:spark:job-server:testJavaJarCreatorPy37 FAILED
FAILURE: Build failed with an exception.
* Where:
Build file '<https://builds.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/runners/spark/job-server/build.gradle'> line: 166
* What went wrong:
Execution failed for task ':runners:spark:job-server:testJavaJarCreatorPy37'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 11m 8s
67 actionable tasks: 51 executed, 15 from cache, 1 up-to-date
Publishing build scan...
https://gradle.com/s/i7sneupiqarcw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org