You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/01/02 12:23:18 UTC
[airflow] branch master updated: Adds timeout to all curl commands
(#13431)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 0909ddf Adds timeout to all curl commands (#13431)
0909ddf is described below
commit 0909ddfd24f22570da1ab1e03484efa90d61382e
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sat Jan 2 13:23:06 2021 +0100
Adds timeout to all curl commands (#13431)
Curl has a sophisticated back-off mechanism when trying to connect
and it causes sometimes that it hangs for a very long time
when first few attempts to connect failed with a 'soft' error.
Similarly, when curl starts transfer after connecting but the
other party hanged, the client curl call might hang as well.
This causes various problems for example sometimes waitig for
images in the ci build gets cancelled because curl command
to check for image fails - example:
https://github.com/apache/airflow/pull/13413/checks?check_run_id=1635401914
This change adds appropriate timeouts to all curl commands we
use in CI/manual operations. In many cases we implemented
retry so the effect will be that those cases will stop happening
but even in no-retry case, failing curl is better than hangs.
---
dev/remove_artifacts.sh | 3 ++-
scripts/ci/libraries/_kind.sh | 8 ++++----
scripts/ci/libraries/_push_pull_remove_images.sh | 3 ++-
scripts/in_container/run_generate_constraints.sh | 2 +-
4 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/dev/remove_artifacts.sh b/dev/remove_artifacts.sh
index 4b3c3f0..fe9c35b 100755
--- a/dev/remove_artifacts.sh
+++ b/dev/remove_artifacts.sh
@@ -44,7 +44,8 @@ GITHUB_TOKEN=${GITHUB_TOKEN}
readonly GITHUB_TOKEN
function github_api_call() {
- curl --silent --location --user "${GITHUB_USER}:${GITHUB_TOKEN}" "$@"
+ curl curl --connect-timeout 60 --max-time 60 \
+ --silent --location --user "${GITHUB_USER}:${GITHUB_TOKEN}" "$@"
}
# A temporary file which receives HTTP response headers.
diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh
index e6be30c..0e4e135 100644
--- a/scripts/ci/libraries/_kind.sh
+++ b/scripts/ci/libraries/_kind.sh
@@ -49,7 +49,7 @@ function kind::make_sure_kubernetes_tools_are_installed() {
echo
echo "Downloading Kind version ${KIND_VERSION}"
repeats::run_with_retry 4 \
- "curl --fail --location '${KIND_URL}' --output '${KIND_BINARY_PATH}'"
+ "curl curl --connect-timeout 60 --max-time 180 --fail --location '${KIND_URL}' --output '${KIND_BINARY_PATH}'"
chmod a+x "${KIND_BINARY_PATH}"
else
echo "Kind version ok"
@@ -65,7 +65,7 @@ function kind::make_sure_kubernetes_tools_are_installed() {
echo
echo "Downloading Kubectl version ${KUBECTL_VERSION}"
repeats::run_with_retry 4 \
- "curl --fail --location '${KUBECTL_URL}' --output '${KUBECTL_BINARY_PATH}'"
+ "curl curl --connect-timeout 60 --max-time 180 --fail --location '${KUBECTL_URL}' --output '${KUBECTL_BINARY_PATH}'"
chmod a+x "${KUBECTL_BINARY_PATH}"
else
echo "Kubectl version ok"
@@ -81,7 +81,7 @@ function kind::make_sure_kubernetes_tools_are_installed() {
echo
echo "Downloading Helm version ${HELM_VERSION}"
repeats::run_with_retry 4 \
- "curl --location '${HELM_URL}' | tar -xvz -O '${SYSTEM}-amd64/helm' >'${HELM_BINARY_PATH}'"
+ "curl curl --connect-timeout 60 --max-time 180 --location '${HELM_URL}' | tar -xvz -O '${SYSTEM}-amd64/helm' >'${HELM_BINARY_PATH}'"
chmod a+x "${HELM_BINARY_PATH}"
else
echo "Helm version ok"
@@ -289,7 +289,7 @@ function kind::wait_for_webserver_healthy() {
num_tries=0
set +e
sleep "${SLEEP_TIME_FOR_HEALTH_CHECK}"
- while ! curl "http://localhost:${FORWARDED_PORT_NUMBER}/health" -s | grep -q healthy; do
+ while ! curl curl --connect-timeout 60 --max-time 60 "http://localhost:${FORWARDED_PORT_NUMBER}/health" -s | grep -q healthy; do
echo
echo "Sleeping ${SLEEP_TIME_FOR_HEALTH_CHECK} while waiting for webserver being ready"
echo
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh
index 1d79c56..da0a077 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -279,7 +279,8 @@ function push_pull_remove_images::wait_for_github_registry_image() {
GITHUB_API_CALL="${github_api_endpoint}/${image_name_in_github_registry}/manifests/${image_tag_in_github_registry}"
while true; do
- curl -X GET "${GITHUB_API_CALL}" -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}" 2>/dev/null > "${OUTPUT_LOG}"
+ curl --connect-timeout 60 --max-time 60 \
+ -X GET "${GITHUB_API_CALL}" -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}" 2>/dev/null > "${OUTPUT_LOG}"
local digest
digest=$(jq '.config.digest' < "${OUTPUT_LOG}")
echo -n "."
diff --git a/scripts/in_container/run_generate_constraints.sh b/scripts/in_container/run_generate_constraints.sh
index 62b2237..60d5368 100755
--- a/scripts/in_container/run_generate_constraints.sh
+++ b/scripts/in_container/run_generate_constraints.sh
@@ -28,7 +28,7 @@ mkdir -pv "${CONSTRAINTS_DIR}"
CONSTRAINTS_LOCATION="https://raw.githubusercontent.com/apache/airflow/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
readonly CONSTRAINTS_LOCATION
-curl "${CONSTRAINTS_LOCATION}" --output "${LATEST_CONSTRAINT_FILE}"
+curl curl --connect-timeout 60 --max-time 60 "${CONSTRAINTS_LOCATION}" --output "${LATEST_CONSTRAINT_FILE}"
echo
echo "Freezing constraints to ${CURRENT_CONSTRAINT_FILE}"