You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by cr...@apache.org on 2017/09/22 17:15:45 UTC
incubator-airflow git commit: [AIRFLOW-1368] Automatically remove
Docker container on exit
Repository: incubator-airflow
Updated Branches:
refs/heads/master 06bd4d3c1 -> 46c86a5cd
[AIRFLOW-1368] Automatically remove Docker container on exit
Closes #2411 from nathanielvarona/docker-operator
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/46c86a5c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/46c86a5c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/46c86a5c
Branch: refs/heads/master
Commit: 46c86a5cd2b69f4f1853280b442e8810b178e6c7
Parents: 06bd4d3
Author: Nathaniel Varona <na...@users.noreply.github.com>
Authored: Fri Sep 22 10:15:02 2017 -0700
Committer: Chris Riccomini <cr...@apache.org>
Committed: Fri Sep 22 10:15:23 2017 -0700
----------------------------------------------------------------------
airflow/operators/docker_operator.py | 14 ++++++++++----
scripts/ci/requirements.txt | 2 +-
setup.py | 2 ++
3 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/46c86a5c/airflow/operators/docker_operator.py
----------------------------------------------------------------------
diff --git a/airflow/operators/docker_operator.py b/airflow/operators/docker_operator.py
index 3011f1c..75a7a39 100644
--- a/airflow/operators/docker_operator.py
+++ b/airflow/operators/docker_operator.py
@@ -18,7 +18,7 @@ from airflow.exceptions import AirflowException
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
from airflow.utils.file import TemporaryDirectory
-from docker import Client, tls
+from docker import APIClient, tls
import ast
@@ -78,6 +78,8 @@ class DockerOperator(BaseOperator):
:type xcom_push: bool
:param xcom_all: Push all the stdout or just the last line. The default is False (last line).
:type xcom_all: bool
+ :param auto_remove: Automatically remove the container when it exits
+ :type auto_remove: bool
"""
template_fields = ('command',)
template_ext = ('.sh', '.bash',)
@@ -105,6 +107,7 @@ class DockerOperator(BaseOperator):
working_dir=None,
xcom_push=False,
xcom_all=False,
+ auto_remove=False,
*args,
**kwargs):
@@ -129,6 +132,7 @@ class DockerOperator(BaseOperator):
self.working_dir = working_dir
self.xcom_push_flag = xcom_push
self.xcom_all = xcom_all
+ self.auto_remove = auto_remove
self.cli = None
self.container = None
@@ -147,7 +151,7 @@ class DockerOperator(BaseOperator):
)
self.docker_url = self.docker_url.replace('tcp://', 'https://')
- self.cli = Client(base_url=self.docker_url, version=self.api_version, tls=tls_config)
+ self.cli = APIClient(base_url=self.docker_url, version=self.api_version, tls=tls_config)
if ':' not in self.image:
image = self.image + ':latest'
@@ -170,8 +174,10 @@ class DockerOperator(BaseOperator):
command=self.get_command(),
cpu_shares=cpu_shares,
environment=self.environment,
- host_config=self.cli.create_host_config(binds=self.volumes,
- network_mode=self.network_mode),
+ host_config=self.cli.create_host_config(
+ binds=self.volumes,
+ network_mode=self.network_mode,
+ auto_remove=self.auto_remove),
image=image,
mem_limit=self.mem_limit,
user=self.user,
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/46c86a5c/scripts/ci/requirements.txt
----------------------------------------------------------------------
diff --git a/scripts/ci/requirements.txt b/scripts/ci/requirements.txt
index d612d6f..0429787 100644
--- a/scripts/ci/requirements.txt
+++ b/scripts/ci/requirements.txt
@@ -28,7 +28,7 @@ cryptography
datadog
dill
distributed
-docker-py
+docker
filechunkio
flake8
flask
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/46c86a5c/setup.py
----------------------------------------------------------------------
diff --git a/setup.py b/setup.py
index a97abfd..46f1b1b 100644
--- a/setup.py
+++ b/setup.py
@@ -125,6 +125,8 @@ doc = [
'Sphinx-PyPI-upload>=0.2.1'
]
docker = ['docker-py>=1.6.0']
+docker = ['docker>=2.4.0']
+druid = ['pydruid>=0.2.1']
emr = ['boto3>=1.0.0']
gcp_api = [
'httplib2',