You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@beam.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2023/10/29 17:14:13 UTC
Build failed in Jenkins: beam_PostCommit_Py_ValCont #11334
See <https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/11334/display/redirect>
Changes:
------------------------------------------
[...truncated 2.28 MB...]
XUNIT_FILE="pytest-$IMAGE_NAME.xml"
# Verify in the root of the repository
test -d sdks/python/container
# Verify docker and gcloud commands exist
command -v docker
command -v gcloud
docker -v
gcloud -v
TAG=$(date +%Y%m%d-%H%M%S%N)
CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME
PREBUILD_SDK_CONTAINER_REGISTRY_PATH=us.gcr.io/$PROJECT/$USER/prebuild_python${PY_VERSION//.}_sdk
echo "Using container $CONTAINER"
echo "Using CPU architecture $ARCH"
if [[ "$ARCH" == "x86" ]]; then
# Verify docker image has been built.
docker images | grep "apache/$IMAGE_NAME" | grep "$SDK_VERSION"
# Tag the docker container.
docker tag "apache/$IMAGE_NAME:$SDK_VERSION" "$CONTAINER:$TAG"
# Push the container
gcloud docker -- push $CONTAINER:$TAG
elif [[ "$ARCH" == "ARM" ]]; then
# Reset the multi-arch Python SDK container image tag.
TAG=$MULTIARCH_TAG
MACHINE_TYPE_ARGS="--machine_type=t2a-standard-1"
else
printf "Please give a valid CPU architecture, either x86 or ARM."
exit 1
fi
WARNING: `gcloud docker` will not be supported for Docker client versions above 18.03.
As an alternative, use `gcloud auth configure-docker` to configure `docker` to
use `gcloud` as a credential helper, then use `docker` as you would for non-GCR
registries, e.g. `docker pull gcr.io/project-id/my-image`. Add
`--verbosity=error` to silence this warning: `gcloud docker
--verbosity=error -- pull gcr.io/project-id/my-image`.
See: https://cloud.google.com/container-registry/docs/support/deprecation-notices#gcloud-docker
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Built container image was not removed. Possibly, it was not not saved locally."
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
# Note: we don't delete the multi-arch containers here because this command only deletes the manifest list with the tag,
# the associated container images can't be deleted because they are not tagged. However, multi-arch containers that are
# older than 6 weeks old are deleted by stale_dataflow_prebuilt_image_cleaner.sh that runs daily.
if [[ "$ARCH" == "x86" ]]; then
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container"
fi
for digest in $(gcloud container images list-tags $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk --format="get(digest)")
do gcloud container images delete $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest --force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o log_cli=True -o log_level=Info -o junit_suite_name=$IMAGE_NAME \
-m=it_validatescontainer \
--numprocesses=1 \
--timeout=1800 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
$MACHINE_TYPE_ARGS \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py310:validatesContainer
function cleanup_container {
# Delete the container locally and remotely
docker rmi $CONTAINER:$TAG || echo "Built container image was not removed. Possibly, it was not not saved locally."
for image in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH)
do docker rmi $image || echo "Failed to remove prebuilt sdk container image"
done
# Note: we don't delete the multi-arch containers here because this command only deletes the manifest list with the tag,
# the associated container images can't be deleted because they are not tagged. However, multi-arch containers that are
# older than 6 weeks old are deleted by stale_dataflow_prebuilt_image_cleaner.sh that runs daily.
if [[ "$ARCH" == "x86" ]]; then
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container"
fi
for digest in $(gcloud container images list-tags $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk --format="get(digest)")
do gcloud container images delete $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk@$digest --force-delete-tags --quiet || echo "Failed to remove prebuilt sdk container image"
done
echo "Removed the container"
}
trap cleanup_container EXIT
echo ">>> Successfully built and push container $CONTAINER"
cd sdks/python
SDK_LOCATION=$2
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
pytest -o log_cli=True -o log_level=Info -o junit_suite_name=$IMAGE_NAME \
-m=it_validatescontainer \
--numprocesses=1 \
--timeout=1800 \
--junitxml=$XUNIT_FILE \
--ignore-glob '.*py3\d?\.py$' \
--log-cli-level=INFO \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--region=$REGION \
--sdk_container_image=$CONTAINER:$TAG \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--output=$GCS_LOCATION/output \
--sdk_location=$SDK_LOCATION \
--num_workers=1 \
$MACHINE_TYPE_ARGS \
--docker_registry_push_url=$PREBUILD_SDK_CONTAINER_REGISTRY_PATH"
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
WARNING: Successfully resolved tag to sha256, but it is recommended to use sha256 directly.
Digests:
- us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:156375f8fd30d99cce1a40c5a831d854baffadd92df7cdcc1e079b91aab09fbd
Associated tags:
- 20231029-165122671991137
Tags:
- us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20231029-165122671991137
Deleted [us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20231029-165122671991137].
Deleted [us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:156375f8fd30d99cce1a40c5a831d854baffadd92df7cdcc1e079b91aab09fbd].
> Task :sdks:python:test-suites:dataflow:py310:validatesContainer
echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"
cleanup_container
WARNING: Successfully resolved tag to sha256, but it is recommended to use sha256 directly.
Digests:
- us.gcr.io/apache-beam-testing/jenkins/beam_python3.10_sdk@sha256:c60408ff6e593b22ec3d698d06d11e5468201fda49a4ee5e9aa57fb5125ed4c3
Associated tags:
- 20231029-165202372314621
Tags:
- us.gcr.io/apache-beam-testing/jenkins/beam_python3.10_sdk:20231029-165202372314621
Deleted [us.gcr.io/apache-beam-testing/jenkins/beam_python3.10_sdk:20231029-165202372314621].
Deleted [us.gcr.io/apache-beam-testing/jenkins/beam_python3.10_sdk@sha256:c60408ff6e593b22ec3d698d06d11e5468201fda49a4ee5e9aa57fb5125ed4c3].
Digests:
- us.gcr.io/apache-beam-testing/jenkins/prebuild_python310_sdk/beam_python_prebuilt_sdk@sha256:b53ec243bc0ca751869c47ea89f2236530f4d8a2cd935069c9f5747aff86bade
Associated tags:
- 62fef284-b5f7-4ee4-a191-af72fa3c22f8
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python310_sdk/beam_python_prebuilt_sdk:62fef284-b5f7-4ee4-a191-af72fa3c22f8].
> Task :sdks:python:test-suites:dataflow:py38:validatesContainer
Digests:
- us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:4ce3d2f2b6ab61a6217492c4bd71bbf0948c2d921cf6b3fca3ad194c2096e4f3
Associated tags:
- 073d4b96-39ba-4c7c-b387-b9fe7c6d0ea6
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:073d4b96-39ba-4c7c-b387-b9fe7c6d0ea6].
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:4ce3d2f2b6ab61a6217492c4bd71bbf0948c2d921cf6b3fca3ad194c2096e4f3].
Digests:
- us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:0e3ae0530271a4c1c48b2e092828243184f129df27a13cf6e50e09cdf41a85d1
Associated tags:
- e2a40307-6909-4a75-a49d-286ae2de1013
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:e2a40307-6909-4a75-a49d-286ae2de1013].
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:0e3ae0530271a4c1c48b2e092828243184f129df27a13cf6e50e09cdf41a85d1].
> Task :sdks:python:test-suites:dataflow:py310:validatesContainer
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python310_sdk/beam_python_prebuilt_sdk@sha256:b53ec243bc0ca751869c47ea89f2236530f4d8a2cd935069c9f5747aff86bade].
Digests:
- us.gcr.io/apache-beam-testing/jenkins/prebuild_python310_sdk/beam_python_prebuilt_sdk@sha256:a4f2a3e77e45db97318d115d7f1415980d72828db130d29ece075669b0f5ac83
Associated tags:
- 4d57cb6b-6f44-444a-a552-7516eaaec85d
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python310_sdk/beam_python_prebuilt_sdk:4d57cb6b-6f44-444a-a552-7516eaaec85d].
Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python310_sdk/beam_python_prebuilt_sdk@sha256:a4f2a3e77e45db97318d115d7f1415980d72828db130d29ece075669b0f5ac83].
FAILURE: Build completed with 3 failures.
1: Task failed with an exception.
-----------
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py311:installGcpTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
==============================================================================
2: Task failed with an exception.
-----------
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py39:installGcpTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
==============================================================================
3: Task failed with an exception.
-----------
* What went wrong:
Execution failed for task ':sdks:python:container:py39:docker'.
> Process 'command 'docker'' finished with non-zero exit value 1
* Try:
> Run with --stacktrace option to get the stack trace.
> Run with --info or --debug option to get more log output.
> Get more help at https://help.gradle.org.
==============================================================================
Deprecated Gradle features were used in this build, making it incompatible with Gradle 9.0.
You can use '--warning-mode all' to show the individual deprecation warnings and determine if they come from your own scripts or plugins.
For more on this, please refer to https://docs.gradle.org/8.4/userguide/command_line_interface.html#sec:command_line_warnings in the Gradle documentation.
BUILD FAILED in 33m 37s
54 actionable tasks: 48 executed, 1 from cache, 5 up-to-date
Publishing build scan...
https://ge.apache.org/s/c2b2ov4txfios
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Jenkins build is back to normal : beam_PostCommit_Py_ValCont #11335
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/11335/display/redirect>
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org