You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@beam.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2021/06/02 06:46:52 UTC
Build failed in Jenkins: beam_PostCommit_Py_ValCont #7850
See <https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/7850/display/redirect?page=changes>
Changes:
[Brian Hulette] Add DataFrame API updates to CHANGES.md
------------------------------------------
[...truncated 108.00 KB...]
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o junit_suite_name=$IMAGE_NAME \
-m="it_validatescontainer" \
--show-capture=no \
--numprocesses=1 \
--timeout=900 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--prebuild_sdk_container_base_image=$CONTAINER:$TAG \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
# Where to store integration test outputs.
GCS_LOCATION=${GCS_LOCATION:-gs://temp-storage-for-end-to-end-tests}
# Project for the container and integration test
PROJECT=${PROJECT:-apache-beam-testing}
REGION=${REGION:-us-central1}
IMAGE_PREFIX="$(grep 'docker_image_default_repo_prefix' gradle.properties | cut -d'=' -f2)"
grep 'docker_image_default_repo_prefix' gradle.properties | cut -d'=' -f2
SDK_VERSION="$(grep 'sdk_version' gradle.properties | cut -d'=' -f2)"
grep 'sdk_version' gradle.properties | cut -d'=' -f2
# Other variables branched by Python version.
if [[ $1 == "python36" ]]; then
IMAGE_NAME="${IMAGE_PREFIX}python3.6_sdk" # Use this to create CONTAINER_IMAGE variable.
CONTAINER_PROJECT="sdks:python:container:py36" # Use this to build container by Gradle.
PY_INTERPRETER="python3.6" # Use this in virtualenv command.
elif [[ $1 == "python37" ]]; then
IMAGE_NAME="${IMAGE_PREFIX}python3.7_sdk" # Use this to create CONTAINER_IMAGE variable.
CONTAINER_PROJECT="sdks:python:container:py37" # Use this to build container by Gradle.
PY_INTERPRETER="python3.7" # Use this in virtualenv command.
elif [[ $1 == "python38" ]]; then
IMAGE_NAME="${IMAGE_PREFIX}python3.8_sdk" # Use this to create CONTAINER_IMAGE variable.
CONTAINER_PROJECT="sdks:python:container:py38" # Use this to build container by Gradle.
PY_INTERPRETER="python3.8" # Use this in virtualenv command.
else
echo "Must set Python version with one of 'python36', 'python37' and 'python38' from commandline."
exit 1
fi
XUNIT_FILE="pytest-$IMAGE_NAME.xml"
# Verify in the root of the repository
test -d sdks/python/container
# Verify docker and gcloud commands exist
command -v docker
command -v gcloud
docker -v
gcloud -v
# Verify docker image has been built.
docker images | grep "apache/$IMAGE_NAME" | grep "$SDK_VERSION"
TAG=$(date +%Y%m%d-%H%M%S%N)
date +%Y%m%d-%H%M%S%N
CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME
PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_$1_sdk
echo "Using container $CONTAINER"
# Tag the docker container.
docker tag "apache/$IMAGE_NAME:$SDK_VERSION" "$CONTAINER:$TAG"
# Push the container.
gcloud docker -- push $CONTAINER:$TAG
WARNING: `gcloud docker` will not be supported for Docker client versions above 18.03.
As an alternative, use `gcloud auth configure-docker` to configure `docker` to
use `gcloud` as a credential helper, then use `docker` as you would for non-GCR
registries, e.g. `docker pull gcr.io/project-id/my-image`. Add
`--verbosity=error` to silence this warning: `gcloud docker
--verbosity=error -- pull gcr.io/project-id/my-image`.
See: https://cloud.google.com/container-registry/docs/support/deprecation-notices#gcloud-docker
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Failed to remove container image"
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container"
for digest in $(gcloud container images list-tags $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk --format="get(digest)")
do gcloud container images delete $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest --force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o junit_suite_name=$IMAGE_NAME \
-m="it_validatescontainer" \
--show-capture=no \
--numprocesses=1 \
--timeout=900 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
--prebuild_sdk_container_base_image=$CONTAINER:$TAG \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py36:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Digests:
- us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:ad586fa43deacfcbb4978c51c81ebd58b15c0968d4e2b88820c4323ccba370e8
Associated tags:
- 20210602-061245836743773
Tags:
- us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20210602-061245836743773
Deleted [us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20210602-061245836743773].
Deleted [us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:ad586fa43deacfcbb4978c51c81ebd58b15c0968d4e2b88820c4323ccba370e8].
gcloud container images list-tags $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk --format="get(digest)"
Digests:
- us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:3c3bee0bee089458eddeb37ca19b75ca56bcc2f164ff9c36d579e1529379ec0c
Associated tags:
- 5af72ebb-5d57-4391-aeb3-e38b11bb5134
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:5af72ebb-5d57-4391-aeb3-e38b11bb5134].
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:3c3bee0bee089458eddeb37ca19b75ca56bcc2f164ff9c36d579e1529379ec0c].
Digests:
- us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:409ea89824f38cecdc9fe27ab455b96ff5743873433545995d8c284fe5819198
Associated tags:
- 4f080b29-0e9a-4dfd-b52b-257fd898747f
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:4f080b29-0e9a-4dfd-b52b-257fd898747f].
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:409ea89824f38cecdc9fe27ab455b96ff5743873433545995d8c284fe5819198].
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Digests:
- us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:3b8afdcb9cf5afd913d239f7ef2b2cdc8d1637f308406ade8031c7dbf98d0e13
Associated tags:
- 20210602-061403657550186
Tags:
- us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20210602-061403657550186
Deleted [us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20210602-061403657550186].
Deleted [us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:3b8afdcb9cf5afd913d239f7ef2b2cdc8d1637f308406ade8031c7dbf98d0e13].
gcloud container images list-tags $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk --format="get(digest)"
Digests:
- us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:d03242a8acf32213015b31fcece96f4aa1a6bb186daa321fd246ba90d1d9ce92
Associated tags:
- cc5ffaa9-72c6-42d1-b1ee-46a89f6610f2
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:cc5ffaa9-72c6-42d1-b1ee-46a89f6610f2].
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:d03242a8acf32213015b31fcece96f4aa1a6bb186daa321fd246ba90d1d9ce92].
Digests:
- us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:bff912e474cfaba4bcf494da90106fce4af7f6c1ac8568f893f1e5345a13f030
Associated tags:
- a2ad97b4-6a78-415e-b966-64b746a62122
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:a2ad97b4-6a78-415e-b966-64b746a62122].
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:bff912e474cfaba4bcf494da90106fce4af7f6c1ac8568f893f1e5345a13f030].
> Task :sdks:python:test-suites:dataflow:py37:validatesContainer
cleanup_container
docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH
Digests:
- us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:e02cb7777d0a5b0440c7b271882ccc75509aacbab0355679d1ba8ff14f605207
Associated tags:
- 20210602-061245135397648
Tags:
- us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20210602-061245135397648
Deleted [us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk:20210602-061245135397648].
Deleted [us.gcr.io/apache-beam-testing/jenkins/beam_python3.7_sdk@sha256:e02cb7777d0a5b0440c7b271882ccc75509aacbab0355679d1ba8ff14f605207].
gcloud container images list-tags $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk --format="get(digest)"
Digests:
- us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:3fc82c663858b6c1846a58c8ee1d67fe16ff2707e61397b397b714ccf5da5436
Associated tags:
- 0581b8ec-899d-4deb-b4c5-f4602aee64db
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:0581b8ec-899d-4deb-b4c5-f4602aee64db].
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:3fc82c663858b6c1846a58c8ee1d67fe16ff2707e61397b397b714ccf5da5436].
Digests:
- us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:95515f898881a0a02cc1443bd710f897e2aaffbe4623679098da7a1ba7c599e4
Associated tags:
- 83f26188-ae06-4af3-87ba-a5908402dd62
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk:83f26188-ae06-4af3-87ba-a5908402dd62].
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python37_sdk/beam_python_prebuilt_sdk@sha256:95515f898881a0a02cc1443bd710f897e2aaffbe4623679098da7a1ba7c599e4].
> Task :sdks:python:test-suites:dataflow:py37:validatesContainer FAILED
FAILURE: Build failed with an exception.
* Where:
Script '<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'> line: 286
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:validatesContainer'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/6.8.3/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 46m 25s
51 actionable tasks: 47 executed, 4 up-to-date
Publishing build scan...
https://gradle.com/s/jzrkdqus5cwtu
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Jenkins build is back to normal : beam_PostCommit_Py_ValCont #7851
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/7851/display/redirect?page=changes>
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org