You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/11/30 01:41:15 UTC
[airflow] 01/01: Update setup.py to get non-conflicting set of
dependencies (#12636)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 8bc40c57181af2b010641d1822d5853daaf07634
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Nov 29 19:45:58 2020 +0100
Update setup.py to get non-conflicting set of dependencies (#12636)
This change upgrades setup.py and setup.cfg to provide non-conflicting
`pip check` valid set of constraints for CI image.
---
BREEZE.rst | 3 ++
CI.rst | 2 +-
CONTRIBUTING.rst | 2 +-
breeze | 2 +
scripts/ci/images/ci_prepare_ci_image_on_ci.sh | 1 -
scripts/ci/images/ci_wait_for_all_ci_images.sh | 36 ++--------------
scripts/ci/images/ci_wait_for_all_prod_images.sh | 38 ++---------------
scripts/ci/images/ci_wait_for_ci_image.sh | 51 +++++++++++++++++++++++
scripts/ci/images/ci_wait_for_prod_image.sh | 52 ++++++++++++++++++++++++
scripts/ci/libraries/_build_images.sh | 43 ++++++++++----------
scripts/ci/libraries/_push_pull_remove_images.sh | 44 +++++++++++++++++---
scripts/ci/selective_ci_checks.sh | 6 +--
setup.py | 44 ++++++++++++++------
13 files changed, 209 insertions(+), 115 deletions(-)
diff --git a/BREEZE.rst b/BREEZE.rst
index f91b598..095fe1b 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -1355,6 +1355,7 @@ This is the current syntax for `./breeze <./breeze>`_:
If you use this flag, automatically --github-registry is enabled.
+
Default: latest.
-v, --verbose
@@ -1508,6 +1509,7 @@ This is the current syntax for `./breeze <./breeze>`_:
If you use this flag, automatically --github-registry is enabled.
+
Default: latest.
-v, --verbose
@@ -2276,6 +2278,7 @@ This is the current syntax for `./breeze <./breeze>`_:
If you use this flag, automatically --github-registry is enabled.
+
Default: latest.
****************************************************************************************************
diff --git a/CI.rst b/CI.rst
index f4b5294..fac9f0f 100644
--- a/CI.rst
+++ b/CI.rst
@@ -253,7 +253,7 @@ You can use those variables when you try to reproduce the build locally.
| Image build variables |
+-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+
| ``UPGRADE_TO_LATEST_CONSTRAINTS`` | false | false | false | Determines whether the build should |
-| | | | (x) | attempt to eagerly upgrade all |
+| | | | (x) | attempt to upgrade all |
| | | | | PIP dependencies to latest ones matching |
| | | | | ``setup.py`` limits. This tries to replicate |
| | | | | the situation of "fresh" user who just installs |
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 61883e7..0c1c9c1 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -321,7 +321,7 @@ Step 4: Prepare PR
the "full tests needed" label is set for your PR. Additional check is set that prevents from
accidental merging of the request until full matrix of tests succeeds for the PR.
- * when your change has "upgrade to latest dependencies" label set, constraints will be automatically
+ * when your change has "upgrade to newer dependencies" label set, constraints will be automatically
upgraded to latest constraints matching your setup.py. This is useful in case you want to force
upgrade to a latest version of dependencies. You can ask committers to set the label for you
when you need it in your PR.
diff --git a/breeze b/breeze
index fe8f038..6f73ad7 100755
--- a/breeze
+++ b/breeze
@@ -1078,6 +1078,7 @@ function breeze::parse_arguments() {
echo
echo "Force pulling the image, using github registry and skip mounting local sources."
echo "This is in order to get the exact same version as used in CI environment for SHA/RUN_ID!."
+ echo "You can specify --skip-mounting-local-sources to not mount local sources. "
echo
export FORCE_PULL_IMAGES="true"
export USE_GITHUB_REGISTRY="true"
@@ -2385,6 +2386,7 @@ function breeze::flag_pull_push_docker_images() {
If you use this flag, automatically --github-registry is enabled.
+
Default: ${_breeze_default_github_image_id:=}.
"
diff --git a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
index e2637c3..4d2a1ce 100755
--- a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
@@ -59,5 +59,4 @@ function build_ci_image_on_ci() {
export CHECK_IMAGE_FOR_REBUILD="false"
}
-
build_ci_image_on_ci
diff --git a/scripts/ci/images/ci_wait_for_all_ci_images.sh b/scripts/ci/images/ci_wait_for_all_ci_images.sh
index edb6b29..2451a88 100755
--- a/scripts/ci/images/ci_wait_for_all_ci_images.sh
+++ b/scripts/ci/images/ci_wait_for_all_ci_images.sh
@@ -15,42 +15,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}"
echo
-echo "Airflow sources: ${AIRFLOW_SOURCES}"
+echo "Waiting for all CI images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}"
echo
-if [[ ${USE_GITHUB_REGISTRY} != "true" || ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} != "true" ]]; then
- echo
- echo "This script should not be called"
- echo "It need both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE to true!"
- echo
- echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY}"
- echo "GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}"
- echo
- exit 1
-fi
-
-echo
-echo "Waiting for all images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}"
-echo
-
-echo
-echo "Check if jq is installed"
-echo
-command -v jq >/dev/null || (echo "ERROR! You must have 'jq' tool installed!" && exit 1)
-
-echo
-echo "The jq version $(jq --version)"
-echo
-
-# shellcheck source=scripts/ci/libraries/_all_libs.sh
-source "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
-
-initialization::initialize_common_environment
-
for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}
do
- export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
- push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_CI_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+ export PYTHON_MAJOR_MINOR_VERSION
+ "$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_ci_image.sh"
done
diff --git a/scripts/ci/images/ci_wait_for_all_prod_images.sh b/scripts/ci/images/ci_wait_for_all_prod_images.sh
index 66196c3..25bfd7c 100755
--- a/scripts/ci/images/ci_wait_for_all_prod_images.sh
+++ b/scripts/ci/images/ci_wait_for_all_prod_images.sh
@@ -15,44 +15,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}"
echo
-echo "Airflow sources: ${AIRFLOW_SOURCES}"
+echo "Waiting for all PROD images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}"
echo
-if [[ ${USE_GITHUB_REGISTRY} != "true" || ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} != "true" ]]; then
- echo
- echo "This script should not be called"
- echo "It need both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE to true!"
- echo
- echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY}"
- echo "GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}"
- echo
- exit 1
-fi
-
-echo
-echo "Waiting for all images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}"
-echo
-
-echo
-echo "Check if jq is installed"
-echo
-command -v jq >/dev/null || (echo "ERROR! You must have 'jq' tool installed!" && exit 1)
-
-echo
-echo "The jq version $(jq --version)"
-echo
-
-# shellcheck source=scripts/ci/libraries/_all_libs.sh
-source "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
-
-initialization::initialize_common_environment
-
for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}
do
- export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
- export AIRFLOW_PROD_BUILD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-build"
- push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_PROD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
- push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_PROD_BUILD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+ export PYTHON_MAJOR_MINOR_VERSION
+ "$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_prod_image.sh"
done
diff --git a/scripts/ci/images/ci_wait_for_ci_image.sh b/scripts/ci/images/ci_wait_for_ci_image.sh
new file mode 100755
index 0000000..4b52b9a
--- /dev/null
+++ b/scripts/ci/images/ci_wait_for_ci_image.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+
+function verify_ci_image_dependencies {
+ echo
+ echo "Checking if Airflow dependencies are non-conflicting in CI image."
+ echo
+
+ push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" \
+ "${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+
+ docker run --rm --entrypoint /bin/bash "${AIRFLOW_CI_IMAGE}" -c 'pip check'
+}
+
+push_pull_remove_images::check_if_github_registry_wait_for_image_enabled
+
+push_pull_remove_images::check_if_jq_installed
+
+build_image::login_to_github_registry_if_needed
+
+export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
+
+echo
+echo "Waiting for image to appear: ${AIRFLOW_CI_IMAGE_NAME}"
+echo
+
+push_pull_remove_images::wait_for_github_registry_image \
+ "${AIRFLOW_CI_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+
+echo
+echo "Verifying the ${AIRFLOW_CI_IMAGE_NAME} image after pulling it"
+echo
+
+verify_ci_image_dependencies
diff --git a/scripts/ci/images/ci_wait_for_prod_image.sh b/scripts/ci/images/ci_wait_for_prod_image.sh
new file mode 100755
index 0000000..e53aec1
--- /dev/null
+++ b/scripts/ci/images/ci_wait_for_prod_image.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+
+function verify_prod_image_dependencies {
+ echo
+ echo "Checking if Airflow dependencies are non-conflicting in PROD image."
+ echo
+
+ push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" \
+ "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+
+ # TODO: remove the | true after we fixed pip check for prod image
+ docker run --rm --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'pip check' || true
+}
+
+push_pull_remove_images::check_if_github_registry_wait_for_image_enabled
+
+push_pull_remove_images::check_if_jq_installed
+
+build_image::login_to_github_registry_if_needed
+
+export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
+
+echo
+echo "Waiting for image to appear: ${AIRFLOW_PROD_IMAGE_NAME}"
+echo
+
+push_pull_remove_images::wait_for_github_registry_image \
+ "${AIRFLOW_PROD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+
+echo
+echo "Verifying the ${AIRFLOW_PROD_IMAGE_NAME} image after pulling it"
+echo
+
+verify_prod_image_dependencies
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index 5bd2d06..8de58db 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -346,13 +346,19 @@ function build_images::get_docker_image_names() {
# File that is touched when the CI image is built for the first time locally
export BUILT_CI_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}"
+
+ # GitHub Registry names must be lowercase :(
+ github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')"
+ export GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_PROD_BASE_TAG}"
+ export GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_PROD_BASE_TAG}-build"
+ export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
+
+ export GITHUB_REGISTRY_AIRFLOW_CI_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_CI_BASE_TAG}"
+ export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
}
-# Prepares all variables needed by the CI build. Depending on the configuration used (python version
-# DockerHub user etc. the variables are set so that other functions can use those variables.
-function build_images::prepare_ci_build() {
- export AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
- export AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
+# If GitHub Registry is used, login to the registry using GITHUB_USERNAME and GITHUB_TOKEN
+function build_image::login_to_github_registry_if_needed() {
if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
if [[ -n ${GITHUB_TOKEN=} ]]; then
echo "${GITHUB_TOKEN}" | docker login \
@@ -360,11 +366,15 @@ function build_images::prepare_ci_build() {
--password-stdin \
"${GITHUB_REGISTRY}"
fi
- # GitHub Registry names must be lowercase :(
- github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')"
- export GITHUB_REGISTRY_AIRFLOW_CI_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_CI_BASE_TAG}"
- export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
fi
+
+}
+
+# Prepares all variables needed by the CI build. Depending on the configuration used (python version
+# DockerHub user etc. the variables are set so that other functions can use those variables.
+function build_images::prepare_ci_build() {
+ export AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
+ export AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
export THE_IMAGE_TYPE="CI"
export IMAGE_DESCRIPTION="Airflow CI"
@@ -375,6 +385,7 @@ function build_images::prepare_ci_build() {
export AIRFLOW_IMAGE="${AIRFLOW_CI_IMAGE}"
readonly AIRFLOW_IMAGE
+ build_image::login_to_github_registry_if_needed
sanity_checks::go_to_airflow_sources
permissions::fix_group_permissions
}
@@ -662,19 +673,7 @@ function build_images::prepare_prod_build() {
export AIRFLOW_IMAGE="${AIRFLOW_PROD_IMAGE}"
readonly AIRFLOW_IMAGE
- if [[ ${USE_GITHUB_REGISTRY="false"} == "true" ]]; then
- if [[ -n ${GITHUB_TOKEN=} ]]; then
- echo "${GITHUB_TOKEN}" | docker login \
- --username "${GITHUB_USERNAME}" \
- --password-stdin \
- "${GITHUB_REGISTRY}"
- fi
- # GitHub Registry names must be lowercase :(
- github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')"
- export GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_PROD_BASE_TAG}"
- export GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/${AIRFLOW_PROD_BASE_TAG}-build"
- export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${github_repository_lowercase}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
- fi
+ build_image::login_to_github_registry_if_needed
AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="${BRANCH_NAME}"
sanity_checks::go_to_airflow_sources
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh
index 7c65db1..216e025 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -264,13 +264,18 @@ function push_pull_remove_images::push_prod_images() {
# waits for an image to be available in the github registry
function push_pull_remove_images::wait_for_github_registry_image() {
+ local github_repository_lowercase
github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')"
- GITHUB_API_ENDPOINT="https://${GITHUB_REGISTRY}/v2/${github_repository_lowercase}"
- IMAGE_NAME="${1}"
- IMAGE_TAG=${2}
- echo "Waiting for ${IMAGE_NAME}:${IMAGE_TAG} image"
+ local github_api_endpoint
+ github_api_endpoint="https://${GITHUB_REGISTRY}/v2/${github_repository_lowercase}"
+ local image_name_in_github_registry="${1}"
+ local image_tag_in_github_registry=${2}
- GITHUB_API_CALL="${GITHUB_API_ENDPOINT}/${IMAGE_NAME}/manifests/${IMAGE_TAG}"
+ echo
+ echo "Waiting for ${GITHUB_REPOSITORY}/${image_name_in_github_registry}:${image_tag_in_github_registry} image"
+ echo
+
+ GITHUB_API_CALL="${github_api_endpoint}/${image_name_in_github_registry}/manifests/${image_tag_in_github_registry}"
while true; do
curl -X GET "${GITHUB_API_CALL}" -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}" 2>/dev/null > "${OUTPUT_LOG}"
local digest
@@ -282,6 +287,33 @@ function push_pull_remove_images::wait_for_github_registry_image() {
fi
sleep 10
done
- verbosity::print_info "Found ${IMAGE_NAME}:${IMAGE_TAG} image"
+ verbosity::print_info "Found ${image_name_in_github_registry}:${image_tag_in_github_registry} image"
verbosity::print_info "Digest: '${digest}'"
}
+
+function push_pull_remove_images::check_if_github_registry_wait_for_image_enabled() {
+ if [[ ${USE_GITHUB_REGISTRY} != "true" || ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} != "true" ]]; then
+ echo
+ echo "This script should not be called"
+ echo "It need both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE to true!"
+ echo
+ echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY}"
+ echo "GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}"
+ echo
+ exit 1
+ else
+ echo
+ echo "Both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE are set to true. Good!"
+ fi
+}
+
+function push_pull_remove_images::check_if_jq_installed() {
+ echo
+ echo "Check if jq is installed"
+ echo
+ command -v jq >/dev/null || (echo "ERROR! You must have 'jq' tool installed!" && exit 1)
+
+ echo
+ echo "The jq version $(jq --version)"
+ echo
+}
diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh
index c87ec41..8696d56 100755
--- a/scripts/ci/selective_ci_checks.sh
+++ b/scripts/ci/selective_ci_checks.sh
@@ -42,14 +42,14 @@ else
FULL_TESTS_NEEDED_LABEL="false"
fi
-if [[ ${PR_LABELS=} == *"upgrade to latest dependencies"* ]]; then
+if [[ ${PR_LABELS=} == *"upgrade to newer dependencies"* ]]; then
echo
- echo "Found the right PR labels in '${PR_LABELS=}': 'upgrade to latest dependencies''"
+ echo "Found the right PR labels in '${PR_LABELS=}': 'upgrade to newer dependencies''"
echo
UPGRADE_TO_LATEST_CONSTRAINTS_LABEL="true"
else
echo
- echo "Did not find the right PR labels in '${PR_LABELS=}': 'upgrade to latest dependencies'"
+ echo "Did not find the right PR labels in '${PR_LABELS=}': 'upgrade to newer dependencies'"
echo
UPGRADE_TO_LATEST_CONSTRAINTS_LABEL="false"
fi
diff --git a/setup.py b/setup.py
index f5f2a53..83e8a98 100644
--- a/setup.py
+++ b/setup.py
@@ -182,11 +182,13 @@ atlas = [
'atlasclient>=0.1.2',
]
aws = [
- 'boto3~=1.10',
+ 'boto3~=1.10,<1.11', # required by snowflake
]
azure_blob_storage = [
'azure-storage>=0.34.0, <0.37.0',
- 'azure-storage-blob<12.0',
+ 'azure-storage-blob<12.0.0;python_version<"3.6"',
+ 'azure-storage-blob;python_version>="3.6"',
+ 'azure-storage-common',
]
azure_container_instances = [
'azure-mgmt-containerinstance>=1.5.0,<2'
@@ -198,6 +200,7 @@ azure_data_lake = [
'azure-datalake-store>=0.0.45'
'azure-mgmt-datalake-store>=0.5.0',
'azure-mgmt-resource>=2.2.0',
+ 'cffi<1.14.0;python_version<"3.0"'
]
azure_secrets = [
'azure-identity>=1.3.1',
@@ -207,7 +210,8 @@ cassandra = [
'cassandra-driver>=3.13.0,<3.21.0',
]
celery = [
- 'celery~=4.3',
+ 'celery~=4.3;python_version>="3.0"',
+ 'celery==4.3.1;python_version<"3.0"',
'flower>=0.7.3, <1.0',
'kombu==4.6.3;python_version<"3.0"',
'tornado>=4.2.0, <6.0', # Dep of flower. Pin to a version that works on Py3.5.2
@@ -222,7 +226,8 @@ cloudant = [
crypto = [
# Cryptography 3.2 for python 2.7 is broken
# https://github.com/pyca/cryptography/issues/5359#issuecomment-727622403
- 'cryptography>=0.9.3,<3.2; python_version<"3.0"',
+ # Snowflake requires <3.0
+ 'cryptography>=0.9.3,<3.0; python_version<"3.0"',
'cryptography>=0.9.3;python_version>="3.0"',
]
dask = [
@@ -260,7 +265,8 @@ flask_oauth = [
'requests-oauthlib==1.1.0',
]
gcp = [
- 'PyOpenSSL',
+ 'PyOpenSSL<20.0.0;python_version<"3.0"',
+ 'PyOpenSSL;python_version>="3.0"',
'google-api-python-client>=1.6.0, <2.0.0',
'google-auth>=1.0.0, <2.0.0',
'google-auth-httplib2>=0.0.1',
@@ -306,7 +312,8 @@ jira = [
kerberos = [
'pykerberos>=1.1.13',
'requests_kerberos>=0.10.0',
- 'thrift_sasl>=0.2.0',
+ 'thrift_sasl>=0.2.0,<0.4.1;python_version<"3.0"',
+ 'thrift_sasl>=0.2.0;python_version>="3.0"',
]
kubernetes = [
'cryptography>=2.0.0',
@@ -336,7 +343,9 @@ papermill = [
'papermill[all]>=1.0.0',
'nteract-scrapbook[all]>=0.2.1',
'pyarrow<1.0.0',
- 'fsspec<0.8.0;python_version=="3.5"'
+ 'fsspec<0.8.0;python_version=="3.5"',
+ 'black==20.8b0;python_version>="3.6"' # we need to limit black version as we have click < 7
+
]
password = [
'bcrypt>=2.0.0',
@@ -355,7 +364,7 @@ qds = [
'qds-sdk>=1.10.4',
]
rabbitmq = [
- 'amqp',
+ 'amqp<5.0.0',
]
redis = [
'redis~=3.2',
@@ -378,6 +387,7 @@ sentry = [
]
slack = [
'slackclient>=1.0.0,<2.0.0',
+ 'websocket-client<0.55.0'
]
snowflake = [
'snowflake-connector-python>=1.5.2',
@@ -421,11 +431,14 @@ devel = [
'click==6.7',
'contextdecorator;python_version<"3.4"',
'coverage',
+ 'docutils>=0.14, <0.16',
+ 'ecdsa<0.15', # Required for moto 1.3.14
'flake8>=3.6.0',
'flake8-colors',
'flaky',
'freezegun',
'gitpython',
+ 'idna<2.9', # Required for moto 1.3.14
'importlib-metadata~=2.0; python_version<"3.8"',
'ipdb',
'jira',
@@ -436,14 +449,15 @@ devel = [
'packaging',
'parameterized',
'paramiko',
+ 'pipdeptree',
'pre-commit',
+ 'pyrsistent<=0.16.0;python_version<"3.0"',
+ 'pyrsistent;python_version>="3.0"',
'pysftp',
'pytest<6.0.0', # FIXME: pylint complaining for pytest.mark.* on v6.0
'pytest-cov',
'pytest-instafail',
- 'pytest-rerunfailures',
'pytest-timeouts',
- 'pytest-xdist',
'pywinrm',
'qds-sdk>=1.9.6',
'requests_mock',
@@ -590,6 +604,8 @@ INSTALL_REQUIREMENTS = [
'colorlog==4.0.2',
'configparser>=3.5.0, <3.6.0',
'croniter>=0.3.17, <0.4',
+ 'cryptography>=0.9.3,<3.0; python_version<"3.0"', # required by snowflake
+ 'cryptography>=0.9.3;python_version>="3.0"',
'dill>=0.2.2, <0.4',
'email-validator',
'enum34~=1.1.6;python_version<"3.4"',
@@ -606,11 +622,12 @@ INSTALL_REQUIREMENTS = [
'graphviz>=0.12',
'gunicorn>=19.5.0, <21.0',
'importlib-metadata~=2.0; python_version<"3.8"',
+ 'importlib_resources~=1.4',
'iso8601>=0.1.12',
'jinja2>=2.10.1, <2.12.0',
'json-merge-patch==0.2',
'jsonschema~=3.0',
- 'lazy_object_proxy~=1.3',
+ 'lazy_object_proxy<1.5.0', # Required to keep pip-check happy with astroid
'markdown>=2.5.2, <3.0',
'marshmallow-sqlalchemy>=0.16.1, <0.24.0;python_version>="3.6"',
'marshmallow-sqlalchemy>=0.16.1, <0.19.0;python_version<"3.6"',
@@ -624,14 +641,15 @@ INSTALL_REQUIREMENTS = [
'python-dateutil>=2.3, <3',
'python-nvd3~=0.15.0',
'python-slugify>=3.0.0,<5.0',
- 'requests>=2.20.0, <3',
+ 'requests>=2.20.0, <2.23.0;python_version<"3.0"', # Required to keep snowflake happy
+ 'requests>=2.20.0, <2.24.0;python_version>="3.0"', # Required to keep snowflake happy
'setproctitle>=1.1.8, <2',
'sqlalchemy~=1.3',
'sqlalchemy_jsonfield==0.8.0;python_version<"3.5"',
'sqlalchemy_jsonfield~=0.9;python_version>="3.5"',
'tabulate>=0.7.5, <0.9',
'tenacity==4.12.0',
- 'thrift>=0.9.2',
+ 'thrift>=0.11.0',
'typing;python_version<"3.5"',
'typing-extensions>=3.7.4;python_version<"3.8"',
'tzlocal>=1.4,<2.0.0',