You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tv...@apache.org on 2023/04/15 01:23:14 UTC
[beam] branch master updated: Add ValidatesContainer tests with installing release candidates(RCs) (#26156)
This is an automated email from the ASF dual-hosted git repository.
tvalentyn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new e86486dd6aa Add ValidatesContainer tests with installing release candidates(RCs) (#26156)
e86486dd6aa is described below
commit e86486dd6aaac6641e5571fd7417be4eea5d7ce5
Author: Anand Inguva <34...@users.noreply.github.com>
AuthorDate: Fri Apr 14 21:23:06 2023 -0400
Add ValidatesContainer tests with installing release candidates(RCs) (#26156)
Co-authored-by: tvalentyn <tv...@users.noreply.github.com>
---
...ommit_Python_ValidatesContainer_Dataflow.groovy | 22 +++++++++++++++
sdks/python/container/Dockerfile | 2 --
sdks/python/container/common.gradle | 3 ++-
sdks/python/container/run_generate_requirements.sh | 31 +++++++++++++---------
sdks/python/test-suites/dataflow/common.gradle | 8 +++++-
5 files changed, 49 insertions(+), 17 deletions(-)
diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
index bc43ecb366f..25409644866 100644
--- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
@@ -43,3 +43,25 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont',
}
}
}
+
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont_with_RC',
+ 'Run Python RC Dataflow ValidatesContainer', 'Google Cloud Dataflow Runner Python ValidatesContainer Tests with RC Dependencies', this) {
+ description('Runs Python ValidatesContainer suite on the Dataflow runner by installing Release Candidates.')
+
+ // Set common parameters.
+ commonJobProperties.setTopLevelMainJobProperties(delegate)
+
+ publishers {
+ archiveJunit('**/pytest*.xml')
+ }
+
+ // Execute shell command to test Python SDK.
+ steps {
+ gradle {
+ rootBuildScriptDir(commonJobProperties.checkoutDir)
+ tasks(':sdks:python:test-suites:dataflow:validatesContainerTests')
+ switches('-PtestRCDependencies=true')
+ commonJobProperties.setGradleSwitches(delegate)
+ }
+ }
+ }
diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile
index 8dcfd5b74dd..ff3b4484950 100644
--- a/sdks/python/container/Dockerfile
+++ b/sdks/python/container/Dockerfile
@@ -109,5 +109,3 @@ COPY --from=third_party_licenses /opt/apache/beam/third_party_licenses /opt/apac
RUN if [ "$pull_licenses" != "true" ] ; then \
rm -rf /opt/apache/beam/third_party_licenses ; \
fi
-
-
diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle
index c10211950f8..48c059ae5ec 100644
--- a/sdks/python/container/common.gradle
+++ b/sdks/python/container/common.gradle
@@ -32,13 +32,14 @@ dependencies {
def generatePythonRequirements = tasks.register("generatePythonRequirements") {
dependsOn ':sdks:python:sdist'
+ def pipExtraOptions = project.hasProperty("testRCDependencies") ? "--pre" : ""
def runScriptsPath = "${rootDir}/sdks/python/container/run_generate_requirements.sh"
doLast {
exec {
executable 'sh'
args '-c', "cd ${rootDir} && ${runScriptsPath} " +
"${project.ext.pythonVersion} " +
- "${files(configurations.sdkSourceTarball.files).singleFile}"
+ "${files(configurations.sdkSourceTarball.files).singleFile} " + "${pipExtraOptions}"
}
}
}
diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh
index 55173bb8cd9..fd222107dba 100755
--- a/sdks/python/container/run_generate_requirements.sh
+++ b/sdks/python/container/run_generate_requirements.sh
@@ -16,7 +16,7 @@
# limitations under the License.
#
-# Generates requirements files, which list PyPI depenedncies to install in
+# Generates requirements files, which list PyPI dependencies to install in
# Apache Beam Python SDK container images. To generate the list,
# we use two sources of information:
# 1) Requirements of Apache Beam itself, as defined by setup.py.
@@ -30,21 +30,25 @@
# You will need Python interpreters for all versions supported by Beam, see:
# https://s.apache.org/beam-python-dev-wiki
-if [[ $# != 2 ]]; then
+if [[ $# -lt 2 ]]; then
printf "Example usage: \n$> ./sdks/python/container/run_generate_requirements.sh 3.8 <sdk_tarball>"
- printf "\n\twhere 3.8 is the Python major.minor version."
+ printf "\n\where 3.8 is the Python major.minor version."
exit 1
fi
PY_VERSION=$1
SDK_TARBALL=$2
+# Use the PIP_EXTRA_OPTIONS environment variable to pass additional flags to the pip install command.
+# For example, you can include the --pre flag in $PIP_EXTRA_OPTIONS to download pre-release versions of packages.
+# Note that you can modify the behavior of the pip install command in this script by passing in your own $PIP_EXTRA_OPTIONS.
+PIP_EXTRA_OPTIONS=$3
-if ! python$PY_VERSION --version > /dev/null 2>&1 ; then
+if ! python"$PY_VERSION" --version > /dev/null 2>&1 ; then
echo "Please install a python${PY_VERSION} interpreter. See s.apache.org/beam-python-dev-wiki for Python installation tips."
exit 1
fi
-if ! python$PY_VERSION -m venv --help > /dev/null 2>&1 ; then
+if ! python"$PY_VERSION" -m venv --help > /dev/null 2>&1 ; then
echo "Your python${PY_VERSION} installation does not have a required venv module. See s.apache.org/beam-python-dev-wiki for Python installation tips."
exit 1
fi
@@ -52,17 +56,18 @@ fi
set -ex
ENV_PATH="$PWD/build/python${PY_VERSION/./}_requirements_gen"
-rm -rf $ENV_PATH 2>/dev/null || true
-python${PY_VERSION} -m venv $ENV_PATH
-source $ENV_PATH/bin/activate
+rm -rf "$ENV_PATH" 2>/dev/null || true
+python"${PY_VERSION}" -m venv "$ENV_PATH"
+source "$ENV_PATH"/bin/activate
pip install --upgrade pip setuptools wheel
# Install gcp extra deps since these deps are commonly used with Apache Beam.
# Install dataframe deps to add have Dataframe support in released images.
# Install test deps since some integration tests need dependencies,
# such as pytest, installed in the runner environment.
-pip install --no-cache-dir $SDK_TARBALL[gcp,dataframe,test]
-pip install --no-cache-dir -r $PWD/sdks/python/container/base_image_requirements_manual.txt
+pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir "$SDK_TARBALL"[gcp,dataframe,test]
+pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"} --no-cache-dir -r "$PWD"/sdks/python/container/base_image_requirements_manual.txt
+
pip uninstall -y apache-beam
echo "Checking for broken dependencies:"
pip check
@@ -71,7 +76,7 @@ pip freeze
PY_IMAGE="py${PY_VERSION//.}"
REQUIREMENTS_FILE=$PWD/sdks/python/container/$PY_IMAGE/base_image_requirements.txt
-cat <<EOT > $REQUIREMENTS_FILE
+cat <<EOT > "$REQUIREMENTS_FILE"
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -98,5 +103,5 @@ cat <<EOT > $REQUIREMENTS_FILE
EOT
# Remove pkg_resources to guard against
# https://stackoverflow.com/questions/39577984/what-is-pkg-resources-0-0-0-in-output-of-pip-freeze-command
-pip freeze | grep -v pkg_resources >> $REQUIREMENTS_FILE
-rm -rf $ENV_PATH
+pip freeze | grep -v pkg_resources >> "$REQUIREMENTS_FILE"
+rm -rf "$ENV_PATH"
diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle
index 2552eac431a..f47b7a5fe90 100644
--- a/sdks/python/test-suites/dataflow/common.gradle
+++ b/sdks/python/test-suites/dataflow/common.gradle
@@ -351,6 +351,12 @@ task chicagoTaxiExample {
task validatesContainer() {
def pyversion = "${project.ext.pythonVersion.replace('.', '')}"
+ if (project.hasProperty("testRCDependencies")) {
+ // Generate a requirements file with pre-release versions for the docker task
+ // if testing with pre-release dependencies.
+ dependsOn ":sdks:python:container:py${pyversion}:generatePythonRequirements"
+ mustRunAfter ":sdks:python:container:py${pyversion}:generatePythonRequirements"
+ }
dependsOn 'initializeForDataflowJob'
dependsOn ":sdks:python:container:py${pyversion}:docker"
def runScriptsPath = "${rootDir}/sdks/python/container/run_validatescontainer.sh"
@@ -457,4 +463,4 @@ project(":sdks:python:test-suites:xlang").ext.xlangTasks.each { taskMetadata ->
],
pytestOptions: basicPytestOpts
)
-}
\ No newline at end of file
+}