You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by GitBox <gi...@apache.org> on 2018/09/21 12:36:11 UTC
[GitHub] Fokko closed pull request #3922: AIRFLOW-2952 Fix Kubernetes CI
Fokko closed pull request #3922: AIRFLOW-2952 Fix Kubernetes CI
URL: https://github.com/apache/incubator-airflow/pull/3922
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.travis.yml b/.travis.yml
index 5bd750453a..dd493363ab 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,23 +26,24 @@ env:
- TRAVIS_CACHE=$HOME/.travis_cache/
matrix:
- TOX_ENV=flake8
- - TOX_ENV=py27-backend_mysql
- - TOX_ENV=py27-backend_sqlite
- - TOX_ENV=py27-backend_postgres
- - TOX_ENV=py35-backend_mysql PYTHON_VERSION=3
- - TOX_ENV=py35-backend_sqlite PYTHON_VERSION=3
- - TOX_ENV=py35-backend_postgres PYTHON_VERSION=3
- - TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.9.0
- - TOX_ENV=py35-backend_postgres KUBERNETES_VERSION=v1.10.0 PYTHON_VERSION=3
+ - TOX_ENV=py27-backend_mysql-env_docker
+ - TOX_ENV=py27-backend_sqlite-env_docker
+ - TOX_ENV=py27-backend_postgres-env_docker
+ - TOX_ENV=py35-backend_mysql-env_docker PYTHON_VERSION=3
+ - TOX_ENV=py35-backend_sqlite-env_docker PYTHON_VERSION=3
+ - TOX_ENV=py35-backend_postgres-env_docker PYTHON_VERSION=3
+ - TOX_ENV=py27-backend_postgres-env_kubernetes KUBERNETES_VERSION=v1.9.0
+ - TOX_ENV=py35-backend_postgres-env_kubernetes KUBERNETES_VERSION=v1.10.0 PYTHON_VERSION=3
+
cache:
directories:
- $HOME/.wheelhouse/
- $HOME/.cache/pip
- $HOME/.travis_cache/
before_install:
- - sudo ls -lh $HOME/.cache/pip/
- - sudo rm -rf $HOME/.cache/pip/* $HOME/.wheelhouse/*
- - sudo chown -R travis:travis $HOME/.cache/pip
+ # Required for K8s v1.10.x. See
+ # https://github.com/kubernetes/kubernetes/issues/61058#issuecomment-372764783
+ - if [ ! -z "$KUBERNETES_VERSION" ]; then sudo mount --make-shared / && sudo service docker restart; fi
install:
# Use recent docker-compose version
- sudo rm /usr/local/bin/docker-compose
@@ -50,5 +51,12 @@ install:
- chmod +x docker-compose
- sudo mv docker-compose /usr/local/bin
- pip install --upgrade pip
+ - if [ ! -z "$KUBERNETES_VERSION" ]; then ./scripts/ci/kubernetes/setup_kubernetes.sh; fi
script:
- - docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml run airflow-testing /app/scripts/ci/run-ci.sh
+ - if [ -z "$KUBERNETES_VERSION" ]; then docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml run airflow-testing /app/scripts/ci/run-ci.sh; fi
+ - if [ ! -z "$KUBERNETES_VERSION" ]; then
+ ./scripts/ci/kubernetes/kube/deploy.sh &&
+ MINIKUBE_IP=$(minikube ip) docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml -f scripts/ci/docker-compose-kubernetes.yml run airflow-testing /app/scripts/ci/run-ci.sh;
+ fi
+before_cache:
+ - sudo chown -R travis:travis $HOME/.cache/pip $HOME/.wheelhouse/
diff --git a/airflow/example_dags_kubernetes/__init__.py b/airflow/example_dags_kubernetes/__init__.py
new file mode 100644
index 0000000000..114d189da1
--- /dev/null
+++ b/airflow/example_dags_kubernetes/__init__.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/airflow/example_dags_kubernetes/example_kubernetes_annotation.py b/airflow/example_dags_kubernetes/example_kubernetes_annotation.py
new file mode 100644
index 0000000000..058baf6990
--- /dev/null
+++ b/airflow/example_dags_kubernetes/example_kubernetes_annotation.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import print_function
+import airflow
+from airflow.operators.python_operator import PythonOperator
+from airflow.models import DAG
+
+args = {
+ 'owner': 'airflow',
+ 'start_date': airflow.utils.dates.days_ago(2)
+}
+
+dag = DAG(
+ dag_id='example_kubernetes_annotation', default_args=args,
+ schedule_interval=None
+)
+
+
+def print_stuff():
+ print("annotated!")
+
+
+# You can use annotations on your kubernetes pods!
+start_task = PythonOperator(
+ task_id="start_task", python_callable=print_stuff, dag=dag,
+ executor_config={
+ "KubernetesExecutor": {
+ "annotations": {"test": "annotation"}
+ }
+ }
+)
diff --git a/scripts/ci/5-run-tests.sh b/scripts/ci/5-run-tests.sh
index 6c6f53d467..8acc78f1a7 100755
--- a/scripts/ci/5-run-tests.sh
+++ b/scripts/ci/5-run-tests.sh
@@ -58,9 +58,11 @@ which airflow > /dev/null || python setup.py develop
# (which contains /usr/local/bin)
sudo ln -sf "${VIRTUAL_ENV}/bin/airflow" /usr/local/bin/
-echo "Initializing the DB"
-yes | airflow initdb
-yes | airflow resetdb
+if [ -z "$KUBERNETES_VERSION" ]; then
+ echo "Initializing the DB"
+ yes | airflow initdb
+ yes | airflow resetdb
+fi
if [ -z "$nose_args" ]; then
nose_args="--with-coverage \
@@ -75,8 +77,10 @@ if [ -z "$nose_args" ]; then
--logging-level=DEBUG"
fi
-# kdc init happens in setup_kdc.sh
-kinit -kt ${KRB5_KTNAME} airflow
+if [ -z "$KUBERNETES_VERSION" ]; then
+ # kdc init happens in setup_kdc.sh
+ kinit -kt ${KRB5_KTNAME} airflow
+fi
# For impersonation tests running on SQLite on Travis, make the database world readable so other
# users can update it
diff --git a/scripts/ci/docker-compose-kubernetes.yml b/scripts/ci/docker-compose-kubernetes.yml
new file mode 100644
index 0000000000..0cdb2df199
--- /dev/null
+++ b/scripts/ci/docker-compose-kubernetes.yml
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "2.2"
+services:
+ airflow-testing:
+ network_mode: host
+ environment:
+ - KUBERNETES_VERSION
+ - MINIKUBE_IP
+ volumes:
+ - /usr/local/bin/kubectl:/usr/local/bin/kubectl
+ - /usr/local/bin/minikube:/usr/local/bin/minikube
+ - ~/.kube:/home/airflow/.kube
+ - ~/.minikube:/home/airflow/.minikube
diff --git a/scripts/ci/kubernetes/docker/airflow-test-env-init.sh b/scripts/ci/kubernetes/docker/airflow-test-env-init.sh
index aa86da72df..33593fd1ca 100755
--- a/scripts/ci/kubernetes/docker/airflow-test-env-init.sh
+++ b/scripts/ci/kubernetes/docker/airflow-test-env-init.sh
@@ -18,7 +18,7 @@
# under the License.
cd /usr/local/lib/python2.7/dist-packages/airflow && \
-cp -R example_dags/* /root/airflow/dags/ && \
+cp -R example_dags_kubernetes/* /root/airflow/dags/ && \
airflow initdb && \
alembic upgrade heads && \
(airflow create_user -u airflow -l airflow -f jon -e airflow@apache.org -r Admin -p airflow || true) && \
diff --git a/scripts/ci/kubernetes/docker/build.sh b/scripts/ci/kubernetes/docker/build.sh
index b93c6b1715..49d0fb87c3 100755
--- a/scripts/ci/kubernetes/docker/build.sh
+++ b/scripts/ci/kubernetes/docker/build.sh
@@ -22,9 +22,13 @@ TAG=${2:-latest}
DIRNAME=$(cd "$(dirname "$0")"; pwd)
AIRFLOW_ROOT="$DIRNAME/../../../.."
-ENVCONFIG=$(minikube docker-env)
-if [ $? -eq 0 ]; then
- eval $ENVCONFIG
+set -e
+
+if [ "${VM_DRIVER:-none}" != "none" ]; then
+ ENVCONFIG=$(minikube docker-env)
+ if [ $? -eq 0 ]; then
+ eval $ENVCONFIG
+ fi
fi
echo "Airflow directory $AIRFLOW_ROOT"
diff --git a/scripts/ci/kubernetes/kube/airflow.yaml b/scripts/ci/kubernetes/kube/airflow.yaml
index 4f451ba44a..1b6e33cec5 100644
--- a/scripts/ci/kubernetes/kube/airflow.yaml
+++ b/scripts/ci/kubernetes/kube/airflow.yaml
@@ -151,4 +151,3 @@ spec:
nodePort: 30809
selector:
name: airflow
-
diff --git a/scripts/ci/kubernetes/kube/configmaps.yaml b/scripts/ci/kubernetes/kube/configmaps.yaml
index c0c7e9b2d0..ab44931e59 100644
--- a/scripts/ci/kubernetes/kube/configmaps.yaml
+++ b/scripts/ci/kubernetes/kube/configmaps.yaml
@@ -27,7 +27,7 @@ data:
logging_level = INFO
executor = KubernetesExecutor
parallelism = 32
- load_examples = True
+ load_examples = False
plugins_folder = /root/airflow/plugins
sql_alchemy_conn = $SQL_ALCHEMY_CONN
diff --git a/scripts/ci/kubernetes/kube/deploy.sh b/scripts/ci/kubernetes/kube/deploy.sh
index a9a42a7a12..75567c9acf 100755
--- a/scripts/ci/kubernetes/kube/deploy.sh
+++ b/scripts/ci/kubernetes/kube/deploy.sh
@@ -17,6 +17,8 @@
# specific language governing permissions and limitations *
# under the License. *
+set -x
+
IMAGE=${1:-airflow/ci}
TAG=${2:-latest}
DIRNAME=$(cd "$(dirname "$0")"; pwd)
diff --git a/scripts/ci/kubernetes/minikube/start_minikube.sh b/scripts/ci/kubernetes/minikube/start_minikube.sh
index c9388e08ff..0803884899 100755
--- a/scripts/ci/kubernetes/minikube/start_minikube.sh
+++ b/scripts/ci/kubernetes/minikube/start_minikube.sh
@@ -19,12 +19,13 @@
#!/usr/bin/env bash
+set -x
_MY_SCRIPT="${BASH_SOURCE[0]}"
_MY_DIR=$(cd "$(dirname "$_MY_SCRIPT")" && pwd)
# Avoids 1.7.x because of https://github.com/kubernetes/minikube/issues/2240
_KUBERNETES_VERSION="${KUBERNETES_VERSION}"
-_MINIKUBE_VERSION="${MINIKUBE_VERSION:-v0.26.0}"
+_MINIKUBE_VERSION="${MINIKUBE_VERSION:-v0.28.2}"
echo "setting up kubernetes ${_KUBERNETES_VERSION}, using minikube ${_MINIKUBE_VERSION}"
@@ -116,7 +117,7 @@ echo Showing kube-system pods
kubectl get -n kube-system pods
(k8s_single_pod_ready -n kube-system -l component=kube-addon-manager) ||
- (_ADDON=$(kubectl get pod -n kube-system -l component=kube-addon-manager
+ (_ADDON=$(kubectl get pod -n kube-system -l component=kube-addon-manager \
--no-headers -o name| cut -d/ -f2);
echo Addon-manager describe:;
kubectl describe pod -n kube-system $_ADDON;
diff --git a/scripts/ci/kubernetes/setup_kubernetes.sh b/scripts/ci/kubernetes/setup_kubernetes.sh
index ea559a02aa..96f8f5d6af 100755
--- a/scripts/ci/kubernetes/setup_kubernetes.sh
+++ b/scripts/ci/kubernetes/setup_kubernetes.sh
@@ -25,6 +25,5 @@ DIRNAME=$(cd "$(dirname "$0")"; pwd)
$DIRNAME/minikube/start_minikube.sh
$DIRNAME/docker/build.sh
-$DIRNAME/kube/deploy.sh
echo "Airflow environment on kubernetes is good to go!"
diff --git a/scripts/ci/run-ci.sh b/scripts/ci/run-ci.sh
index f2815bbd95..1a65bf1ef6 100755
--- a/scripts/ci/run-ci.sh
+++ b/scripts/ci/run-ci.sh
@@ -24,7 +24,7 @@ DIRNAME=$(cd "$(dirname "$0")"; pwd)
AIRFLOW_ROOT="$DIRNAME/../.."
# Fix file permissions
-sudo chown -R airflow.airflow . $HOME/.wheelhouse/ $HOME/.cache/pip
+sudo chown -R airflow.airflow . $HOME/.cache $HOME/.wheelhouse/ $HOME/.cache/pip $HOME/.kube $HOME/.minikube
if [[ $PYTHON_VERSION == '3' ]]; then
PIP=pip3
@@ -41,7 +41,13 @@ if [ -z "$KUBERNETES_VERSION" ];
then
tox -e $TOX_ENV
else
- KUBERNETES_VERSION=${KUBERNETES_VERSION} $DIRNAME/kubernetes/setup_kubernetes.sh && \
+ # This script runs inside a container, the path of the kubernetes certificate
+ # is /home/travis/.minikube/client.crt but the user in the container is `airflow`
+ if [ ! -d /home/travis ]; then
+ sudo mkdir -p /home/travis
+ fi
+ sudo ln -s /home/airflow/.minikube /home/travis/.minikube
+
tox -e $TOX_ENV -- tests.contrib.minikube \
--with-coverage \
--cover-erase \
diff --git a/tests/contrib/minikube/test_kubernetes_executor.py b/tests/contrib/minikube/test_kubernetes_executor.py
index 45d4124d07..23bdfaec8d 100644
--- a/tests/contrib/minikube/test_kubernetes_executor.py
+++ b/tests/contrib/minikube/test_kubernetes_executor.py
@@ -16,6 +16,7 @@
# under the License.
+import os
import unittest
from subprocess import check_call, check_output
import requests.exceptions
@@ -25,18 +26,25 @@
import re
try:
- check_call(["kubectl", "get", "pods"])
+ check_call(["/usr/local/bin/kubectl", "get", "pods"])
except Exception as e:
- raise unittest.SkipTest(
- "Kubernetes integration tests require a minikube cluster;"
- "Skipping tests {}".format(e)
- )
+ if os.environ.get('KUBERNETES_VERSION'):
+ raise e
+ else:
+ raise unittest.SkipTest(
+ "Kubernetes integration tests require a minikube cluster;"
+ "Skipping tests {}".format(e)
+ )
def get_minikube_host():
- host_ip = check_output(['minikube', 'ip'])
- if six.PY3:
- host_ip = host_ip.decode('UTF-8')
+ if "MINIKUBE_IP" in os.environ:
+ host_ip = os.environ['MINIKUBE_IP']
+ else:
+ host_ip = check_output(['/usr/local/bin/minikube', 'ip'])
+ if six.PY3:
+ host_ip = host_ip.decode('UTF-8')
+
host = '{}:30809'.format(host_ip.strip())
return host
diff --git a/tests/contrib/minikube/test_kubernetes_pod_operator.py b/tests/contrib/minikube/test_kubernetes_pod_operator.py
index 595d7aa8b4..f39fcdb03d 100644
--- a/tests/contrib/minikube/test_kubernetes_pod_operator.py
+++ b/tests/contrib/minikube/test_kubernetes_pod_operator.py
@@ -29,12 +29,15 @@
from airflow.contrib.kubernetes.volume import Volume
try:
- check_call(["kubectl", "get", "pods"])
+ check_call(["/usr/local/bin/kubectl", "get", "pods"])
except Exception as e:
- raise unittest.SkipTest(
- "Kubernetes integration tests require a minikube cluster;"
- "Skipping tests {}".format(e)
- )
+ if os.environ.get('KUBERNETES_VERSION'):
+ raise e
+ else:
+ raise unittest.SkipTest(
+ "Kubernetes integration tests require a minikube cluster;"
+ "Skipping tests {}".format(e)
+ )
class KubernetesPodOperatorTest(unittest.TestCase):
diff --git a/tests/core.py b/tests/core.py
index b937178a9e..a135857e37 100644
--- a/tests/core.py
+++ b/tests/core.py
@@ -97,7 +97,8 @@ def reset(dag_id=TEST_DAG_ID):
configuration.conf.load_test_config()
-reset()
+if os.environ.get('KUBERNETES_VERSION') is None:
+ reset()
class OperatorSubclass(BaseOperator):
diff --git a/tox.ini b/tox.ini
index c4b74a1e55..1ba7d84fea 100644
--- a/tox.ini
+++ b/tox.ini
@@ -17,7 +17,7 @@
# under the License.
[tox]
-envlist = flake8,{py27,py35}-backend_{mysql,sqlite,postgres}
+envlist = flake8,{py27,py35}-backend_{mysql,sqlite,postgres}-env_{docker,kubernetes}
skipsdist = True
[global]
@@ -57,10 +57,10 @@ passenv = *
commands =
pip wheel --progress-bar off -w {homedir}/.wheelhouse -f {homedir}/.wheelhouse -e .[devel_ci]
pip install --progress-bar off --find-links={homedir}/.wheelhouse --no-index -e .[devel_ci]
- {toxinidir}/scripts/ci/1-setup-env.sh
- {toxinidir}/scripts/ci/2-setup-kdc.sh
- {toxinidir}/scripts/ci/3-setup-databases.sh
- {toxinidir}/scripts/ci/4-load-data.sh
+ env_docker: {toxinidir}/scripts/ci/1-setup-env.sh
+ env_docker: {toxinidir}/scripts/ci/2-setup-kdc.sh
+ env_docker: {toxinidir}/scripts/ci/3-setup-databases.sh
+ env_docker: {toxinidir}/scripts/ci/4-load-data.sh
{toxinidir}/scripts/ci/5-run-tests.sh []
{toxinidir}/scripts/ci/6-check-license.sh
codecov -e TOXENV
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services