You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@beam.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2022/01/03 06:11:28 UTC
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #3760
See <https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/3760/display/redirect>
Changes:
------------------------------------------
[...truncated 40.41 KB...]
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
PS1="${_OLD_VIRTUAL_PS1:-}"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1:-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/2022703441">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
# could use `if (set -u; : $PYTHONHOME) ;` in bash
if [ -n "${PYTHONHOME:-}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1:-}"
if [ "x(2022703441) " != x ] ; then
PS1="(2022703441) ${PS1:-}"
else
if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then
# special case for Aspen magic directories
# see http://www.zetadev.com/software/aspen/
PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1"
else
PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1"
fi
fi
export PS1
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
pip install --retries 10 --upgrade pip setuptools wheel
Collecting pip
Using cached pip-21.3.1-py3-none-any.whl (1.7 MB)
Collecting setuptools
Using cached setuptools-60.2.0-py3-none-any.whl (953 kB)
Collecting wheel
Using cached wheel-0.37.1-py2.py3-none-any.whl (35 kB)
Installing collected packages: pip, setuptools, wheel
Attempting uninstall: pip
Found existing installation: pip 20.1.1
Uninstalling pip-20.1.1:
Successfully uninstalled pip-20.1.1
Attempting uninstall: setuptools
Found existing installation: setuptools 47.1.0
Uninstalling setuptools-47.1.0:
Successfully uninstalled setuptools-47.1.0
Successfully installed pip-21.3.1 setuptools-60.2.0 wheel-0.37.1
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Preparing metadata (setup.py): started
Preparing metadata (setup.py): finished with status 'done'
Collecting crcmod<2.0,>=1.7
Using cached crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Collecting orjson<4.0
Using cached orjson-3.6.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (247 kB)
Collecting dill<0.3.2,>=0.3.1.1
Using cached dill-0.3.1.1-py3-none-any.whl
Collecting cloudpickle<3,>=2.0.0
Using cached cloudpickle-2.0.0-py3-none-any.whl (25 kB)
Collecting fastavro<2,>=0.21.4
Using cached fastavro-1.4.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.3 MB)
Collecting grpcio<2,>=1.29.0
Using cached grpcio-1.43.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.1 MB)
Collecting hdfs<3.0.0,>=2.1.0
Using cached hdfs-2.6.0-py3-none-any.whl (33 kB)
Collecting httplib2<0.20.0,>=0.8
Using cached httplib2-0.19.1-py3-none-any.whl (95 kB)
Collecting numpy<1.22.0,>=1.14.3
Using cached numpy-1.21.5-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.12.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (508 kB)
Collecting oauth2client<5,>=2.0.1
Using cached oauth2client-4.1.3-py2.py3-none-any.whl (98 kB)
Collecting protobuf<4,>=3.12.2
Using cached protobuf-3.19.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)
Collecting proto-plus<2,>=1.7.1
Using cached proto_plus-1.19.8-py3-none-any.whl (45 kB)
Collecting pyarrow<7.0.0,>=0.15.1
Using cached pyarrow-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25.6 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.2-py2.py3-none-any.whl (21 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)
Collecting pytz>=2018.3
Using cached pytz-2021.3-py2.py3-none-any.whl (503 kB)
Collecting requests<3.0.0,>=2.24.0
Using cached requests-2.26.0-py2.py3-none-any.whl (62 kB)
Collecting typing-extensions<4,>=3.7.0
Using cached typing_extensions-3.10.0.2-py3-none-any.whl (26 kB)
Collecting six>=1.5.2
Using cached six-1.16.0-py2.py3-none-any.whl (11 kB)
Collecting docopt
Using cached docopt-0.6.2-py2.py3-none-any.whl
Collecting pyparsing<3,>=2.4.2
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.8-py3-none-any.whl (39 kB)
Collecting charset-normalizer~=2.0.0
Using cached charset_normalizer-2.0.9-py3-none-any.whl (39 kB)
Collecting idna<4,>=2.5
Using cached idna-3.3-py3-none-any.whl (61 kB)
Collecting urllib3<1.27,>=1.21.1
Using cached urllib3-1.26.7-py2.py3-none-any.whl (138 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2021.10.8-py2.py3-none-any.whl (149 kB)
Installing collected packages: urllib3, pyparsing, pyasn1, idna, charset-normalizer, certifi, six, rsa, requests, pyasn1-modules, protobuf, numpy, httplib2, docopt, typing-extensions, pytz, python-dateutil, pymongo, pydot, pyarrow, proto-plus, orjson, oauth2client, hdfs, grpcio, fastavro, dill, crcmod, cloudpickle, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam-2.37.0.dev0 certifi-2021.10.8 charset-normalizer-2.0.9 cloudpickle-2.0.0 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-1.4.8 grpcio-1.43.0 hdfs-2.6.0 httplib2-0.19.1 idna-3.3 numpy-1.21.5 oauth2client-4.1.3 orjson-3.6.5 proto-plus-1.19.8 protobuf-3.19.1 pyarrow-6.0.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.2 pymongo-3.12.3 pyparsing-2.4.7 python-dateutil-2.8.2 pytz-2021.3 requests-2.26.0 rsa-4.8 six-1.16.0 typing-extensions-3.10.0.2 urllib3-1.26.7
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR="test-pipeline-${RUNNER}-$(date +%Y%m%d-%H%M%S).jar"
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_options "docker_container_image=$PYTHON_CONTAINER_IMAGE" \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.ClassNotFoundException: org.apache.spark.streaming.api.java.JavaStreamingListener
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 600, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0103061124-602a50d9_8f130d7d-6bef-4f3a-84d7-7ff0f4e8df20 failed in state FAILED: java.lang.ClassNotFoundException: org.apache.spark.streaming.api.java.JavaStreamingListener
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :sdks:python:test-suites:portable:py37:testJavaJarCreatorSparkRunner FAILED
FAILURE: Build failed with an exception.
* Where:
Script '<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/test-suites/portable/common.gradle'> line: 335
* What went wrong:
Execution failed for task ':sdks:python:test-suites:portable:py37:testJavaJarCreatorSparkRunner'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.3.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 10m 54s
73 actionable tasks: 42 executed, 27 from cache, 4 up-to-date
Publishing build scan...
https://gradle.com/s/j3tdydrvpd37i
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Jenkins build is back to normal : beam_PostCommit_PortableJar_Spark #3767
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/3767/display/redirect?page=changes>
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #3766
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/3766/display/redirect?page=changes>
Changes:
[noreply] Merge pull request #16404: [BEAM-13586] Fix NPE in
------------------------------------------
[...truncated 37.30 KB...]
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
PS1="${_OLD_VIRTUAL_PS1:-}"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1:-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/2022703441">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
# could use `if (set -u; : $PYTHONHOME) ;` in bash
if [ -n "${PYTHONHOME:-}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1:-}"
if [ "x(2022703441) " != x ] ; then
PS1="(2022703441) ${PS1:-}"
else
if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then
# special case for Aspen magic directories
# see http://www.zetadev.com/software/aspen/
PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1"
else
PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1"
fi
fi
export PS1
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
pip install --retries 10 --upgrade pip setuptools wheel
Collecting pip
Using cached pip-21.3.1-py3-none-any.whl (1.7 MB)
Collecting setuptools
Using cached setuptools-60.2.0-py3-none-any.whl (953 kB)
Collecting wheel
Using cached wheel-0.37.1-py2.py3-none-any.whl (35 kB)
Installing collected packages: pip, setuptools, wheel
Attempting uninstall: pip
Found existing installation: pip 20.1.1
Uninstalling pip-20.1.1:
Successfully uninstalled pip-20.1.1
Attempting uninstall: setuptools
Found existing installation: setuptools 47.1.0
Uninstalling setuptools-47.1.0:
Successfully uninstalled setuptools-47.1.0
Successfully installed pip-21.3.1 setuptools-60.2.0 wheel-0.37.1
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Preparing metadata (setup.py): started
Preparing metadata (setup.py): finished with status 'done'
Collecting crcmod<2.0,>=1.7
Using cached crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Collecting orjson<4.0
Using cached orjson-3.6.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (247 kB)
Collecting dill<0.3.2,>=0.3.1.1
Using cached dill-0.3.1.1-py3-none-any.whl
Collecting cloudpickle<3,>=2.0.0
Using cached cloudpickle-2.0.0-py3-none-any.whl (25 kB)
Collecting fastavro<2,>=0.21.4
Using cached fastavro-1.4.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.3 MB)
Collecting grpcio<2,>=1.29.0
Using cached grpcio-1.43.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.1 MB)
Collecting hdfs<3.0.0,>=2.1.0
Using cached hdfs-2.6.0-py3-none-any.whl (33 kB)
Collecting httplib2<0.20.0,>=0.8
Using cached httplib2-0.19.1-py3-none-any.whl (95 kB)
Collecting numpy<1.22.0,>=1.14.3
Using cached numpy-1.21.5-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.12.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (508 kB)
Collecting oauth2client<5,>=2.0.1
Using cached oauth2client-4.1.3-py2.py3-none-any.whl (98 kB)
Collecting protobuf<4,>=3.12.2
Using cached protobuf-3.19.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)
Collecting proto-plus<2,>=1.7.1
Using cached proto_plus-1.19.8-py3-none-any.whl (45 kB)
Collecting pyarrow<7.0.0,>=0.15.1
Using cached pyarrow-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25.6 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.2-py2.py3-none-any.whl (21 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)
Collecting pytz>=2018.3
Using cached pytz-2021.3-py2.py3-none-any.whl (503 kB)
Collecting requests<3.0.0,>=2.24.0
Using cached requests-2.27.0-py2.py3-none-any.whl (63 kB)
Collecting typing-extensions<4,>=3.7.0
Using cached typing_extensions-3.10.0.2-py3-none-any.whl (26 kB)
Collecting six>=1.5.2
Using cached six-1.16.0-py2.py3-none-any.whl (11 kB)
Collecting docopt
Using cached docopt-0.6.2-py2.py3-none-any.whl
Collecting pyparsing<3,>=2.4.2
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.8-py3-none-any.whl (39 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting idna<4,>=2.5
Using cached idna-3.3-py3-none-any.whl (61 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2021.10.8-py2.py3-none-any.whl (149 kB)
Collecting charset-normalizer~=2.0.0
Using cached charset_normalizer-2.0.9-py3-none-any.whl (39 kB)
Collecting urllib3<1.27,>=1.21.1
Using cached urllib3-1.26.7-py2.py3-none-any.whl (138 kB)
Installing collected packages: urllib3, pyparsing, pyasn1, idna, charset-normalizer, certifi, six, rsa, requests, pyasn1-modules, protobuf, numpy, httplib2, docopt, typing-extensions, pytz, python-dateutil, pymongo, pydot, pyarrow, proto-plus, orjson, oauth2client, hdfs, grpcio, fastavro, dill, crcmod, cloudpickle, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam-2.37.0.dev0 certifi-2021.10.8 charset-normalizer-2.0.9 cloudpickle-2.0.0 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-1.4.8 grpcio-1.43.0 hdfs-2.6.0 httplib2-0.19.1 idna-3.3 numpy-1.21.5 oauth2client-4.1.3 orjson-3.6.5 proto-plus-1.19.8 protobuf-3.19.1 pyarrow-6.0.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.2 pymongo-3.12.3 pyparsing-2.4.7 python-dateutil-2.8.2 pytz-2021.3 requests-2.27.0 rsa-4.8 six-1.16.0 typing-extensions-3.10.0.2 urllib3-1.26.7
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR="test-pipeline-${RUNNER}-$(date +%Y%m%d-%H%M%S).jar"
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_options "docker_container_image=$PYTHON_CONTAINER_IMAGE" \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.ClassNotFoundException: org.apache.spark.streaming.api.java.JavaStreamingListener
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 600, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0104181136-1e6547b6_cfc660f1-721e-47b9-b902-9e82415a2e52 failed in state FAILED: java.lang.ClassNotFoundException: org.apache.spark.streaming.api.java.JavaStreamingListener
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :sdks:python:test-suites:portable:py37:testJavaJarCreatorSparkRunner FAILED
FAILURE: Build failed with an exception.
* Where:
Script '<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/test-suites/portable/common.gradle'> line: 335
* What went wrong:
Execution failed for task ':sdks:python:test-suites:portable:py37:testJavaJarCreatorSparkRunner'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.3.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 11m 8s
73 actionable tasks: 44 executed, 25 from cache, 4 up-to-date
Publishing build scan...
https://gradle.com/s/eyghv2r57m7ly
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #3765
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/3765/display/redirect>
Changes:
------------------------------------------
[...truncated 36.19 KB...]
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
PS1="${_OLD_VIRTUAL_PS1:-}"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1:-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/2022703441">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
# could use `if (set -u; : $PYTHONHOME) ;` in bash
if [ -n "${PYTHONHOME:-}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1:-}"
if [ "x(2022703441) " != x ] ; then
PS1="(2022703441) ${PS1:-}"
else
if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then
# special case for Aspen magic directories
# see http://www.zetadev.com/software/aspen/
PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1"
else
PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1"
fi
fi
export PS1
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
pip install --retries 10 --upgrade pip setuptools wheel
Collecting pip
Using cached pip-21.3.1-py3-none-any.whl (1.7 MB)
Collecting setuptools
Using cached setuptools-60.2.0-py3-none-any.whl (953 kB)
Collecting wheel
Using cached wheel-0.37.1-py2.py3-none-any.whl (35 kB)
Installing collected packages: pip, setuptools, wheel
Attempting uninstall: pip
Found existing installation: pip 20.1.1
Uninstalling pip-20.1.1:
Successfully uninstalled pip-20.1.1
Attempting uninstall: setuptools
Found existing installation: setuptools 47.1.0
Uninstalling setuptools-47.1.0:
Successfully uninstalled setuptools-47.1.0
Successfully installed pip-21.3.1 setuptools-60.2.0 wheel-0.37.1
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Preparing metadata (setup.py): started
Preparing metadata (setup.py): finished with status 'done'
Collecting crcmod<2.0,>=1.7
Using cached crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Collecting orjson<4.0
Using cached orjson-3.6.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (247 kB)
Collecting dill<0.3.2,>=0.3.1.1
Using cached dill-0.3.1.1-py3-none-any.whl
Collecting cloudpickle<3,>=2.0.0
Using cached cloudpickle-2.0.0-py3-none-any.whl (25 kB)
Collecting fastavro<2,>=0.21.4
Using cached fastavro-1.4.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.3 MB)
Collecting grpcio<2,>=1.29.0
Using cached grpcio-1.43.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.1 MB)
Collecting hdfs<3.0.0,>=2.1.0
Using cached hdfs-2.6.0-py3-none-any.whl (33 kB)
Collecting httplib2<0.20.0,>=0.8
Using cached httplib2-0.19.1-py3-none-any.whl (95 kB)
Collecting numpy<1.22.0,>=1.14.3
Using cached numpy-1.21.5-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.12.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (508 kB)
Collecting oauth2client<5,>=2.0.1
Using cached oauth2client-4.1.3-py2.py3-none-any.whl (98 kB)
Collecting protobuf<4,>=3.12.2
Using cached protobuf-3.19.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)
Collecting proto-plus<2,>=1.7.1
Using cached proto_plus-1.19.8-py3-none-any.whl (45 kB)
Collecting pyarrow<7.0.0,>=0.15.1
Using cached pyarrow-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25.6 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.2-py2.py3-none-any.whl (21 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)
Collecting pytz>=2018.3
Using cached pytz-2021.3-py2.py3-none-any.whl (503 kB)
Collecting requests<3.0.0,>=2.24.0
Using cached requests-2.27.0-py2.py3-none-any.whl (63 kB)
Collecting typing-extensions<4,>=3.7.0
Using cached typing_extensions-3.10.0.2-py3-none-any.whl (26 kB)
Collecting six>=1.5.2
Using cached six-1.16.0-py2.py3-none-any.whl (11 kB)
Collecting docopt
Using cached docopt-0.6.2-py2.py3-none-any.whl
Collecting pyparsing<3,>=2.4.2
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.8-py3-none-any.whl (39 kB)
Collecting urllib3<1.27,>=1.21.1
Using cached urllib3-1.26.7-py2.py3-none-any.whl (138 kB)
Collecting idna<4,>=2.5
Using cached idna-3.3-py3-none-any.whl (61 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2021.10.8-py2.py3-none-any.whl (149 kB)
Collecting charset-normalizer~=2.0.0
Using cached charset_normalizer-2.0.9-py3-none-any.whl (39 kB)
Installing collected packages: urllib3, pyparsing, pyasn1, idna, charset-normalizer, certifi, six, rsa, requests, pyasn1-modules, protobuf, numpy, httplib2, docopt, typing-extensions, pytz, python-dateutil, pymongo, pydot, pyarrow, proto-plus, orjson, oauth2client, hdfs, grpcio, fastavro, dill, crcmod, cloudpickle, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam-2.37.0.dev0 certifi-2021.10.8 charset-normalizer-2.0.9 cloudpickle-2.0.0 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-1.4.8 grpcio-1.43.0 hdfs-2.6.0 httplib2-0.19.1 idna-3.3 numpy-1.21.5 oauth2client-4.1.3 orjson-3.6.5 proto-plus-1.19.8 protobuf-3.19.1 pyarrow-6.0.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.2 pymongo-3.12.3 pyparsing-2.4.7 python-dateutil-2.8.2 pytz-2021.3 requests-2.27.0 rsa-4.8 six-1.16.0 typing-extensions-3.10.0.2 urllib3-1.26.7
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR="test-pipeline-${RUNNER}-$(date +%Y%m%d-%H%M%S).jar"
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_options "docker_container_image=$PYTHON_CONTAINER_IMAGE" \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.ClassNotFoundException: org.apache.spark.streaming.api.java.JavaStreamingListener
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 600, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0104121229-3cbaa626_cd49b265-8912-4b79-8ee5-6844a3b9b950 failed in state FAILED: java.lang.ClassNotFoundException: org.apache.spark.streaming.api.java.JavaStreamingListener
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
>>> FAILURE
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :sdks:python:test-suites:portable:py37:testJavaJarCreatorSparkRunner FAILED
FAILURE: Build failed with an exception.
* Where:
Script '<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/test-suites/portable/common.gradle'> line: 335
* What went wrong:
Execution failed for task ':sdks:python:test-suites:portable:py37:testJavaJarCreatorSparkRunner'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.3.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 12m
73 actionable tasks: 42 executed, 27 from cache, 4 up-to-date
Publishing build scan...
https://gradle.com/s/34ruevrlsacfk
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #3764
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/3764/display/redirect>
Changes:
------------------------------------------
[...truncated 39.65 KB...]
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
PS1="${_OLD_VIRTUAL_PS1:-}"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1:-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/2022703441">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
# could use `if (set -u; : $PYTHONHOME) ;` in bash
if [ -n "${PYTHONHOME:-}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1:-}"
if [ "x(2022703441) " != x ] ; then
PS1="(2022703441) ${PS1:-}"
else
if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then
# special case for Aspen magic directories
# see http://www.zetadev.com/software/aspen/
PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1"
else
PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1"
fi
fi
export PS1
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
pip install --retries 10 --upgrade pip setuptools wheel
Collecting pip
Using cached pip-21.3.1-py3-none-any.whl (1.7 MB)
Collecting setuptools
Using cached setuptools-60.2.0-py3-none-any.whl (953 kB)
Collecting wheel
Using cached wheel-0.37.1-py2.py3-none-any.whl (35 kB)
Installing collected packages: pip, setuptools, wheel
Attempting uninstall: pip
Found existing installation: pip 20.1.1
Uninstalling pip-20.1.1:
Successfully uninstalled pip-20.1.1
Attempting uninstall: setuptools
Found existing installation: setuptools 47.1.0
Uninstalling setuptools-47.1.0:
Successfully uninstalled setuptools-47.1.0
Successfully installed pip-21.3.1 setuptools-60.2.0 wheel-0.37.1
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Preparing metadata (setup.py): started
Preparing metadata (setup.py): finished with status 'done'
Collecting crcmod<2.0,>=1.7
Using cached crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Collecting orjson<4.0
Using cached orjson-3.6.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (247 kB)
Collecting dill<0.3.2,>=0.3.1.1
Using cached dill-0.3.1.1-py3-none-any.whl
Collecting cloudpickle<3,>=2.0.0
Using cached cloudpickle-2.0.0-py3-none-any.whl (25 kB)
Collecting fastavro<2,>=0.21.4
Using cached fastavro-1.4.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.3 MB)
Collecting grpcio<2,>=1.29.0
Using cached grpcio-1.43.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.1 MB)
Collecting hdfs<3.0.0,>=2.1.0
Using cached hdfs-2.6.0-py3-none-any.whl (33 kB)
Collecting httplib2<0.20.0,>=0.8
Using cached httplib2-0.19.1-py3-none-any.whl (95 kB)
Collecting numpy<1.22.0,>=1.14.3
Using cached numpy-1.21.5-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.12.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (508 kB)
Collecting oauth2client<5,>=2.0.1
Using cached oauth2client-4.1.3-py2.py3-none-any.whl (98 kB)
Collecting protobuf<4,>=3.12.2
Using cached protobuf-3.19.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)
Collecting proto-plus<2,>=1.7.1
Using cached proto_plus-1.19.8-py3-none-any.whl (45 kB)
Collecting pyarrow<7.0.0,>=0.15.1
Using cached pyarrow-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25.6 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.2-py2.py3-none-any.whl (21 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)
Collecting pytz>=2018.3
Using cached pytz-2021.3-py2.py3-none-any.whl (503 kB)
Collecting requests<3.0.0,>=2.24.0
Using cached requests-2.27.0-py2.py3-none-any.whl (63 kB)
Collecting typing-extensions<4,>=3.7.0
Using cached typing_extensions-3.10.0.2-py3-none-any.whl (26 kB)
Collecting six>=1.5.2
Using cached six-1.16.0-py2.py3-none-any.whl (11 kB)
Collecting docopt
Using cached docopt-0.6.2-py2.py3-none-any.whl
Collecting pyparsing<3,>=2.4.2
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.8-py3-none-any.whl (39 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting charset-normalizer~=2.0.0
Using cached charset_normalizer-2.0.9-py3-none-any.whl (39 kB)
Collecting urllib3<1.27,>=1.21.1
Using cached urllib3-1.26.7-py2.py3-none-any.whl (138 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2021.10.8-py2.py3-none-any.whl (149 kB)
Collecting idna<4,>=2.5
Using cached idna-3.3-py3-none-any.whl (61 kB)
Installing collected packages: urllib3, pyparsing, pyasn1, idna, charset-normalizer, certifi, six, rsa, requests, pyasn1-modules, protobuf, numpy, httplib2, docopt, typing-extensions, pytz, python-dateutil, pymongo, pydot, pyarrow, proto-plus, orjson, oauth2client, hdfs, grpcio, fastavro, dill, crcmod, cloudpickle, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam-2.37.0.dev0 certifi-2021.10.8 charset-normalizer-2.0.9 cloudpickle-2.0.0 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-1.4.8 grpcio-1.43.0 hdfs-2.6.0 httplib2-0.19.1 idna-3.3 numpy-1.21.5 oauth2client-4.1.3 orjson-3.6.5 proto-plus-1.19.8 protobuf-3.19.1 pyarrow-6.0.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.2 pymongo-3.12.3 pyparsing-2.4.7 python-dateutil-2.8.2 pytz-2021.3 requests-2.27.0 rsa-4.8 six-1.16.0 typing-extensions-3.10.0.2 urllib3-1.26.7
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR="test-pipeline-${RUNNER}-$(date +%Y%m%d-%H%M%S).jar"
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_options "docker_container_image=$PYTHON_CONTAINER_IMAGE" \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.ClassNotFoundException: org.apache.spark.streaming.api.java.JavaStreamingListener
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 600, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0104061235-de842615_30014bc3-c3d4-4fce-aa18-401216b3d171 failed in state FAILED: java.lang.ClassNotFoundException: org.apache.spark.streaming.api.java.JavaStreamingListener
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
>>> FAILURE
> Task :sdks:python:test-suites:portable:py37:testJavaJarCreatorSparkRunner FAILED
FAILURE: Build failed with an exception.
* Where:
Script '<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/test-suites/portable/common.gradle'> line: 335
* What went wrong:
Execution failed for task ':sdks:python:test-suites:portable:py37:testJavaJarCreatorSparkRunner'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.3.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 12m 6s
73 actionable tasks: 42 executed, 27 from cache, 4 up-to-date
Publishing build scan...
https://gradle.com/s/uymwj46ztg5ou
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #3763
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/3763/display/redirect>
Changes:
------------------------------------------
[...truncated 39.88 KB...]
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
PS1="${_OLD_VIRTUAL_PS1:-}"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1:-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/2022703441">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
# could use `if (set -u; : $PYTHONHOME) ;` in bash
if [ -n "${PYTHONHOME:-}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1:-}"
if [ "x(2022703441) " != x ] ; then
PS1="(2022703441) ${PS1:-}"
else
if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then
# special case for Aspen magic directories
# see http://www.zetadev.com/software/aspen/
PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1"
else
PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1"
fi
fi
export PS1
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
pip install --retries 10 --upgrade pip setuptools wheel
Collecting pip
Using cached pip-21.3.1-py3-none-any.whl (1.7 MB)
Collecting setuptools
Using cached setuptools-60.2.0-py3-none-any.whl (953 kB)
Collecting wheel
Using cached wheel-0.37.1-py2.py3-none-any.whl (35 kB)
Installing collected packages: pip, setuptools, wheel
Attempting uninstall: pip
Found existing installation: pip 20.1.1
Uninstalling pip-20.1.1:
Successfully uninstalled pip-20.1.1
Attempting uninstall: setuptools
Found existing installation: setuptools 47.1.0
Uninstalling setuptools-47.1.0:
Successfully uninstalled setuptools-47.1.0
Successfully installed pip-21.3.1 setuptools-60.2.0 wheel-0.37.1
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Preparing metadata (setup.py): started
Preparing metadata (setup.py): finished with status 'done'
Collecting crcmod<2.0,>=1.7
Using cached crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Collecting orjson<4.0
Using cached orjson-3.6.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (247 kB)
Collecting dill<0.3.2,>=0.3.1.1
Using cached dill-0.3.1.1-py3-none-any.whl
Collecting cloudpickle<3,>=2.0.0
Using cached cloudpickle-2.0.0-py3-none-any.whl (25 kB)
Collecting fastavro<2,>=0.21.4
Using cached fastavro-1.4.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.3 MB)
Collecting grpcio<2,>=1.29.0
Using cached grpcio-1.43.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.1 MB)
Collecting hdfs<3.0.0,>=2.1.0
Using cached hdfs-2.6.0-py3-none-any.whl (33 kB)
Collecting httplib2<0.20.0,>=0.8
Using cached httplib2-0.19.1-py3-none-any.whl (95 kB)
Collecting numpy<1.22.0,>=1.14.3
Using cached numpy-1.21.5-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.12.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (508 kB)
Collecting oauth2client<5,>=2.0.1
Using cached oauth2client-4.1.3-py2.py3-none-any.whl (98 kB)
Collecting protobuf<4,>=3.12.2
Using cached protobuf-3.19.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)
Collecting proto-plus<2,>=1.7.1
Using cached proto_plus-1.19.8-py3-none-any.whl (45 kB)
Collecting pyarrow<7.0.0,>=0.15.1
Using cached pyarrow-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25.6 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.2-py2.py3-none-any.whl (21 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)
Collecting pytz>=2018.3
Using cached pytz-2021.3-py2.py3-none-any.whl (503 kB)
Collecting requests<3.0.0,>=2.24.0
Using cached requests-2.27.0-py2.py3-none-any.whl (63 kB)
Collecting typing-extensions<4,>=3.7.0
Using cached typing_extensions-3.10.0.2-py3-none-any.whl (26 kB)
Collecting six>=1.5.2
Using cached six-1.16.0-py2.py3-none-any.whl (11 kB)
Collecting docopt
Using cached docopt-0.6.2-py2.py3-none-any.whl
Collecting pyparsing<3,>=2.4.2
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.8-py3-none-any.whl (39 kB)
Collecting idna<4,>=2.5
Using cached idna-3.3-py3-none-any.whl (61 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2021.10.8-py2.py3-none-any.whl (149 kB)
Collecting charset-normalizer~=2.0.0
Using cached charset_normalizer-2.0.9-py3-none-any.whl (39 kB)
Collecting urllib3<1.27,>=1.21.1
Using cached urllib3-1.26.7-py2.py3-none-any.whl (138 kB)
Installing collected packages: urllib3, pyparsing, pyasn1, idna, charset-normalizer, certifi, six, rsa, requests, pyasn1-modules, protobuf, numpy, httplib2, docopt, typing-extensions, pytz, python-dateutil, pymongo, pydot, pyarrow, proto-plus, orjson, oauth2client, hdfs, grpcio, fastavro, dill, crcmod, cloudpickle, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam-2.37.0.dev0 certifi-2021.10.8 charset-normalizer-2.0.9 cloudpickle-2.0.0 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-1.4.8 grpcio-1.43.0 hdfs-2.6.0 httplib2-0.19.1 idna-3.3 numpy-1.21.5 oauth2client-4.1.3 orjson-3.6.5 proto-plus-1.19.8 protobuf-3.19.1 pyarrow-6.0.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.2 pymongo-3.12.3 pyparsing-2.4.7 python-dateutil-2.8.2 pytz-2021.3 requests-2.27.0 rsa-4.8 six-1.16.0 typing-extensions-3.10.0.2 urllib3-1.26.7
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR="test-pipeline-${RUNNER}-$(date +%Y%m%d-%H%M%S).jar"
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_options "docker_container_image=$PYTHON_CONTAINER_IMAGE" \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.ClassNotFoundException: org.apache.spark.streaming.api.java.JavaStreamingListener
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 600, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0104001440-37172bdd_fc726797-0af5-4605-bc44-0151d3d84141 failed in state FAILED: java.lang.ClassNotFoundException: org.apache.spark.streaming.api.java.JavaStreamingListener
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
>>> FAILURE
else
echo ">>> FAILURE"
fi
exit $TEST_EXIT_CODE
> Task :sdks:python:test-suites:portable:py37:testJavaJarCreatorSparkRunner FAILED
FAILURE: Build failed with an exception.
* Where:
Script '<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/test-suites/portable/common.gradle'> line: 335
* What went wrong:
Execution failed for task ':sdks:python:test-suites:portable:py37:testJavaJarCreatorSparkRunner'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.3.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 12s
73 actionable tasks: 42 executed, 27 from cache, 4 up-to-date
Publishing build scan...
https://gradle.com/s/b5f4fsybbwsmy
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #3762
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/3762/display/redirect?page=changes>
Changes:
[mmack] [BEAM-13509] Stop sharing SQS client between readers of same source.
[mmack] [BEAM-13587] Attempt to load AWS region from default provider chain in
------------------------------------------
[...truncated 35.51 KB...]
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
PS1="${_OLD_VIRTUAL_PS1:-}"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1:-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/2022703441">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
# could use `if (set -u; : $PYTHONHOME) ;` in bash
if [ -n "${PYTHONHOME:-}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1:-}"
if [ "x(2022703441) " != x ] ; then
PS1="(2022703441) ${PS1:-}"
else
if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then
# special case for Aspen magic directories
# see http://www.zetadev.com/software/aspen/
PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1"
else
PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1"
fi
fi
export PS1
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
pip install --retries 10 --upgrade pip setuptools wheel
Collecting pip
Using cached pip-21.3.1-py3-none-any.whl (1.7 MB)
Collecting setuptools
Using cached setuptools-60.2.0-py3-none-any.whl (953 kB)
Collecting wheel
Using cached wheel-0.37.1-py2.py3-none-any.whl (35 kB)
Installing collected packages: pip, setuptools, wheel
Attempting uninstall: pip
Found existing installation: pip 20.1.1
Uninstalling pip-20.1.1:
Successfully uninstalled pip-20.1.1
Attempting uninstall: setuptools
Found existing installation: setuptools 47.1.0
Uninstalling setuptools-47.1.0:
Successfully uninstalled setuptools-47.1.0
Successfully installed pip-21.3.1 setuptools-60.2.0 wheel-0.37.1
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Preparing metadata (setup.py): started
Preparing metadata (setup.py): finished with status 'done'
Collecting crcmod<2.0,>=1.7
Using cached crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Collecting orjson<4.0
Using cached orjson-3.6.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (247 kB)
Collecting dill<0.3.2,>=0.3.1.1
Using cached dill-0.3.1.1-py3-none-any.whl
Collecting cloudpickle<3,>=2.0.0
Using cached cloudpickle-2.0.0-py3-none-any.whl (25 kB)
Collecting fastavro<2,>=0.21.4
Using cached fastavro-1.4.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.3 MB)
Collecting grpcio<2,>=1.29.0
Using cached grpcio-1.43.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.1 MB)
Collecting hdfs<3.0.0,>=2.1.0
Using cached hdfs-2.6.0-py3-none-any.whl (33 kB)
Collecting httplib2<0.20.0,>=0.8
Using cached httplib2-0.19.1-py3-none-any.whl (95 kB)
Collecting numpy<1.22.0,>=1.14.3
Using cached numpy-1.21.5-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.12.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (508 kB)
Collecting oauth2client<5,>=2.0.1
Using cached oauth2client-4.1.3-py2.py3-none-any.whl (98 kB)
Collecting protobuf<4,>=3.12.2
Using cached protobuf-3.19.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)
Collecting proto-plus<2,>=1.7.1
Using cached proto_plus-1.19.8-py3-none-any.whl (45 kB)
Collecting pyarrow<7.0.0,>=0.15.1
Using cached pyarrow-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25.6 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.2-py2.py3-none-any.whl (21 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)
Collecting pytz>=2018.3
Using cached pytz-2021.3-py2.py3-none-any.whl (503 kB)
Collecting requests<3.0.0,>=2.24.0
Using cached requests-2.27.0-py2.py3-none-any.whl (63 kB)
Collecting typing-extensions<4,>=3.7.0
Using cached typing_extensions-3.10.0.2-py3-none-any.whl (26 kB)
Collecting six>=1.5.2
Using cached six-1.16.0-py2.py3-none-any.whl (11 kB)
Collecting docopt
Using cached docopt-0.6.2-py2.py3-none-any.whl
Collecting pyparsing<3,>=2.4.2
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.8-py3-none-any.whl (39 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting idna<4,>=2.5
Using cached idna-3.3-py3-none-any.whl (61 kB)
Collecting urllib3<1.27,>=1.21.1
Using cached urllib3-1.26.7-py2.py3-none-any.whl (138 kB)
Collecting charset-normalizer~=2.0.0
Using cached charset_normalizer-2.0.9-py3-none-any.whl (39 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2021.10.8-py2.py3-none-any.whl (149 kB)
Installing collected packages: urllib3, pyparsing, pyasn1, idna, charset-normalizer, certifi, six, rsa, requests, pyasn1-modules, protobuf, numpy, httplib2, docopt, typing-extensions, pytz, python-dateutil, pymongo, pydot, pyarrow, proto-plus, orjson, oauth2client, hdfs, grpcio, fastavro, dill, crcmod, cloudpickle, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam-2.37.0.dev0 certifi-2021.10.8 charset-normalizer-2.0.9 cloudpickle-2.0.0 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-1.4.8 grpcio-1.43.0 hdfs-2.6.0 httplib2-0.19.1 idna-3.3 numpy-1.21.5 oauth2client-4.1.3 orjson-3.6.5 proto-plus-1.19.8 protobuf-3.19.1 pyarrow-6.0.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.2 pymongo-3.12.3 pyparsing-2.4.7 python-dateutil-2.8.2 pytz-2021.3 requests-2.27.0 rsa-4.8 six-1.16.0 typing-extensions-3.10.0.2 urllib3-1.26.7
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR="test-pipeline-${RUNNER}-$(date +%Y%m%d-%H%M%S).jar"
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_options "docker_container_image=$PYTHON_CONTAINER_IMAGE" \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.ClassNotFoundException: org.apache.spark.streaming.api.java.JavaStreamingListener
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 600, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0103181207-8b0fc088_7fab1f4f-026a-44da-9251-4aa6442167a0 failed in state FAILED: java.lang.ClassNotFoundException: org.apache.spark.streaming.api.java.JavaStreamingListener
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
>>> FAILURE
fi
exit $TEST_EXIT_CODE
> Task :sdks:python:test-suites:portable:py37:testJavaJarCreatorSparkRunner FAILED
FAILURE: Build failed with an exception.
* Where:
Script '<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/test-suites/portable/common.gradle'> line: 335
* What went wrong:
Execution failed for task ':sdks:python:test-suites:portable:py37:testJavaJarCreatorSparkRunner'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.3.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 11m 35s
73 actionable tasks: 44 executed, 25 from cache, 4 up-to-date
Publishing build scan...
https://gradle.com/s/2tylqwmlbsi4i
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PostCommit_PortableJar_Spark #3761
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/3761/display/redirect>
Changes:
------------------------------------------
[...truncated 37.54 KB...]
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
PS1="${_OLD_VIRTUAL_PS1:-}"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1:-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/build/gradleenv/2022703441">
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
# could use `if (set -u; : $PYTHONHOME) ;` in bash
if [ -n "${PYTHONHOME:-}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1:-}"
if [ "x(2022703441) " != x ] ; then
PS1="(2022703441) ${PS1:-}"
else
if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then
# special case for Aspen magic directories
# see http://www.zetadev.com/software/aspen/
PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1"
else
PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1"
fi
fi
export PS1
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
hash -r
fi
pip install --retries 10 --upgrade pip setuptools wheel
Collecting pip
Using cached pip-21.3.1-py3-none-any.whl (1.7 MB)
Collecting setuptools
Using cached setuptools-60.2.0-py3-none-any.whl (953 kB)
Collecting wheel
Using cached wheel-0.37.1-py2.py3-none-any.whl (35 kB)
Installing collected packages: pip, setuptools, wheel
Attempting uninstall: pip
Found existing installation: pip 20.1.1
Uninstalling pip-20.1.1:
Successfully uninstalled pip-20.1.1
Attempting uninstall: setuptools
Found existing installation: setuptools 47.1.0
Uninstalling setuptools-47.1.0:
Successfully uninstalled setuptools-47.1.0
Successfully installed pip-21.3.1 setuptools-60.2.0 wheel-0.37.1
pip install --retries 10 -e $PYTHON_ROOT_DIR
Obtaining file://<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python>
Preparing metadata (setup.py): started
Preparing metadata (setup.py): finished with status 'done'
Collecting crcmod<2.0,>=1.7
Using cached crcmod-1.7-cp37-cp37m-linux_x86_64.whl
Collecting orjson<4.0
Using cached orjson-3.6.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (247 kB)
Collecting dill<0.3.2,>=0.3.1.1
Using cached dill-0.3.1.1-py3-none-any.whl
Collecting cloudpickle<3,>=2.0.0
Using cached cloudpickle-2.0.0-py3-none-any.whl (25 kB)
Collecting fastavro<2,>=0.21.4
Using cached fastavro-1.4.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.3 MB)
Collecting grpcio<2,>=1.29.0
Using cached grpcio-1.43.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.1 MB)
Collecting hdfs<3.0.0,>=2.1.0
Using cached hdfs-2.6.0-py3-none-any.whl (33 kB)
Collecting httplib2<0.20.0,>=0.8
Using cached httplib2-0.19.1-py3-none-any.whl (95 kB)
Collecting numpy<1.22.0,>=1.14.3
Using cached numpy-1.21.5-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)
Collecting pymongo<4.0.0,>=3.8.0
Using cached pymongo-3.12.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (508 kB)
Collecting oauth2client<5,>=2.0.1
Using cached oauth2client-4.1.3-py2.py3-none-any.whl (98 kB)
Collecting protobuf<4,>=3.12.2
Using cached protobuf-3.19.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)
Collecting proto-plus<2,>=1.7.1
Using cached proto_plus-1.19.8-py3-none-any.whl (45 kB)
Collecting pyarrow<7.0.0,>=0.15.1
Using cached pyarrow-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25.6 MB)
Collecting pydot<2,>=1.2.0
Using cached pydot-1.4.2-py2.py3-none-any.whl (21 kB)
Collecting python-dateutil<3,>=2.8.0
Using cached python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)
Collecting pytz>=2018.3
Using cached pytz-2021.3-py2.py3-none-any.whl (503 kB)
Collecting requests<3.0.0,>=2.24.0
Using cached requests-2.26.0-py2.py3-none-any.whl (62 kB)
Collecting typing-extensions<4,>=3.7.0
Using cached typing_extensions-3.10.0.2-py3-none-any.whl (26 kB)
Collecting six>=1.5.2
Using cached six-1.16.0-py2.py3-none-any.whl (11 kB)
Collecting docopt
Using cached docopt-0.6.2-py2.py3-none-any.whl
Collecting pyparsing<3,>=2.4.2
Using cached pyparsing-2.4.7-py2.py3-none-any.whl (67 kB)
Collecting pyasn1>=0.1.7
Using cached pyasn1-0.4.8-py2.py3-none-any.whl (77 kB)
Collecting rsa>=3.1.4
Using cached rsa-4.8-py3-none-any.whl (39 kB)
Collecting pyasn1-modules>=0.0.5
Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)
Collecting urllib3<1.27,>=1.21.1
Using cached urllib3-1.26.7-py2.py3-none-any.whl (138 kB)
Collecting certifi>=2017.4.17
Using cached certifi-2021.10.8-py2.py3-none-any.whl (149 kB)
Collecting charset-normalizer~=2.0.0
Using cached charset_normalizer-2.0.9-py3-none-any.whl (39 kB)
Collecting idna<4,>=2.5
Using cached idna-3.3-py3-none-any.whl (61 kB)
Installing collected packages: urllib3, pyparsing, pyasn1, idna, charset-normalizer, certifi, six, rsa, requests, pyasn1-modules, protobuf, numpy, httplib2, docopt, typing-extensions, pytz, python-dateutil, pymongo, pydot, pyarrow, proto-plus, orjson, oauth2client, hdfs, grpcio, fastavro, dill, crcmod, cloudpickle, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam-2.37.0.dev0 certifi-2021.10.8 charset-normalizer-2.0.9 cloudpickle-2.0.0 crcmod-1.7 dill-0.3.1.1 docopt-0.6.2 fastavro-1.4.8 grpcio-1.43.0 hdfs-2.6.0 httplib2-0.19.1 idna-3.3 numpy-1.21.5 oauth2client-4.1.3 orjson-3.6.5 proto-plus-1.19.8 protobuf-3.19.1 pyarrow-6.0.1 pyasn1-0.4.8 pyasn1-modules-0.2.8 pydot-1.4.2 pymongo-3.12.3 pyparsing-2.4.7 python-dateutil-2.8.2 pytz-2021.3 requests-2.26.0 rsa-4.8 six-1.16.0 typing-extensions-3.10.0.2 urllib3-1.26.7
PIPELINE_PY="
import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
from apache_beam.transforms import Create
from apache_beam.transforms import Map
# To test that our main session is getting plumbed through artifact staging
# correctly, create a global variable. If the main session is not plumbed
# through properly, global_var will be undefined and the pipeline will fail.
global_var = 1
pipeline_options = PipelineOptions()
pipeline_options.view_as(SetupOptions).save_main_session = True
pipeline = beam.Pipeline(options=pipeline_options)
pcoll = (pipeline
| Create([0, 1, 2])
| Map(lambda x: x + global_var))
assert_that(pcoll, equal_to([1, 2, 3]))
result = pipeline.run()
result.wait_until_finish()
"
if [[ "$RUNNER" = "FlinkRunner" ]]; then
INPUT_JAR_ARG="flink_job_server_jar"
else
INPUT_JAR_ARG="spark_job_server_jar"
fi
# Create the jar
OUTPUT_JAR="test-pipeline-${RUNNER}-$(date +%Y%m%d-%H%M%S).jar"
date +%Y%m%d-%H%M%S
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
--output_executable_path $OUTPUT_JAR \
--parallelism 1 \
--sdk_worker_parallelism 1 \
--environment_type DOCKER \
--environment_options "docker_container_image=$PYTHON_CONTAINER_IMAGE" \
) || TEST_EXIT_CODE=$? # don't fail fast here; clean up before exiting
WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter.
WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--parallelism', '1']
ERROR:root:java.lang.ClassNotFoundException: org.apache.spark.streaming.api.java.JavaStreamingListener
Traceback (most recent call last):
File "<string>", line 24, in <module>
File "<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/apache_beam/runners/portability/portable_runner.py",> line 600, in wait_until_finish
raise self._runtime_exception
RuntimeError: Pipeline BeamApp-jenkins-0103121222-6b9af5d6_d5c6ec9a-f740-48da-9c88-5d04a7dbe1c3 failed in state FAILED: java.lang.ClassNotFoundException: org.apache.spark.streaming.api.java.JavaStreamingListener
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
# Execute the jar
java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$?
fi
rm -rf $ENV_DIR
rm -f $OUTPUT_JAR
if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then
echo ">>> SUCCESS"
else
echo ">>> FAILURE"
fi
>>> FAILURE
exit $TEST_EXIT_CODE
> Task :sdks:python:test-suites:portable:py37:testJavaJarCreatorSparkRunner FAILED
FAILURE: Build failed with an exception.
* Where:
Script '<https://ci-beam.apache.org/job/beam_PostCommit_PortableJar_Spark/ws/src/sdks/python/test-suites/portable/common.gradle'> line: 335
* What went wrong:
Execution failed for task ':sdks:python:test-suites:portable:py37:testJavaJarCreatorSparkRunner'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 8.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
See https://docs.gradle.org/7.3.2/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 11m 53s
73 actionable tasks: 42 executed, 27 from cache, 4 up-to-date
Publishing build scan...
https://gradle.com/s/mqqda6dz7m3zs
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org