You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by "ASF GitHub Bot (JIRA)" <ji...@apache.org> on 2018/08/23 09:41:00 UTC
[jira] [Commented] (AIRFLOW-1380) DockerOperator does not work with
most recent version of Docker client
[ https://issues.apache.org/jira/browse/AIRFLOW-1380?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16589972#comment-16589972 ]
ASF GitHub Bot commented on AIRFLOW-1380:
-----------------------------------------
benjaminsims closed pull request #2417: [AIRFLOW-1380] Update import to work with latest Docker client
URL: https://github.com/apache/incubator-airflow/pull/2417
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/airflow/hooks/docker_hook.py b/airflow/hooks/docker_hook.py
index a570292ead..3512641c49 100644
--- a/airflow/hooks/docker_hook.py
+++ b/airflow/hooks/docker_hook.py
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from docker import Client
+from docker import APIClient as Client
from docker.errors import APIError
from airflow.exceptions import AirflowException
diff --git a/airflow/operators/docker_operator.py b/airflow/operators/docker_operator.py
index 38edc8b4d7..fd540f1b8a 100644
--- a/airflow/operators/docker_operator.py
+++ b/airflow/operators/docker_operator.py
@@ -19,7 +19,7 @@
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
from airflow.utils.file import TemporaryDirectory
-from docker import Client, tls
+from docker import APIClient as Client, tls
import ast
@@ -53,6 +53,9 @@ class DockerOperator(BaseOperator):
:type environment: dict
:param force_pull: Pull the docker image on every run. Default is false.
:type force_pull: bool
+ :param host_tmp_dir: Specify the location of the temporary directory on the host which will
+ be mapped to tmp_dir
+ :type host_tmp_dir: str
:param mem_limit: Maximum amount of memory the container can use. Either a float value, which
represents the limit in bytes, or a string like ``128m`` or ``1g``.
:type mem_limit: float or str
@@ -101,6 +104,7 @@ def __init__(
docker_url='unix://var/run/docker.sock',
environment=None,
force_pull=False,
+ host_tmp_dir=None,
mem_limit=None,
network_mode=None,
tls_ca_cert=None,
@@ -125,6 +129,7 @@ def __init__(
self.docker_url = docker_url
self.environment = environment or {}
self.force_pull = force_pull
+ self.host_tmp_dir = host_tmp_dir
self.image = image
self.mem_limit = mem_limit
self.network_mode = network_mode
@@ -179,7 +184,7 @@ def execute(self, context):
cpu_shares = int(round(self.cpus * 1024))
- with TemporaryDirectory(prefix='airflowtmp') as host_tmp_dir:
+ with TemporaryDirectory(prefix='airflowtmp', dir=self.host_tmp_dir) as host_tmp_dir:
self.environment['AIRFLOW_TMP_DIR'] = self.tmp_dir
self.volumes.append('{0}:{1}'.format(host_tmp_dir, self.tmp_dir))
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000..e0f58b87c5
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,4 @@
+cryptography==2.0.3
+docker==2.5.1
+jsonschema==2.6.0
+tqdm==4.18.0
diff --git a/tests/operators/docker_operator.py b/tests/operators/docker_operator.py
index a12b6f829f..4b83c1f22e 100644
--- a/tests/operators/docker_operator.py
+++ b/tests/operators/docker_operator.py
@@ -17,8 +17,8 @@
try:
from airflow.operators.docker_operator import DockerOperator
+ from docker.client import APIClient as Client
from airflow.hooks.docker_hook import DockerHook
- from docker import Client
except ImportError:
pass
@@ -51,6 +51,7 @@ def test_execute(self, client_class_mock, mkdtemp_mock):
client_class_mock.return_value = client_mock
operator = DockerOperator(api_version='1.19', command='env', environment={'UNIT': 'TEST'},
+ host_tmp_dir='/host/airflow',
image='ubuntu:latest', network_mode='bridge', owner='unittest',
task_id='unittest', volumes=['/host/path:/container/path'],
working_dir='/container/path')
@@ -69,6 +70,8 @@ def test_execute(self, client_class_mock, mkdtemp_mock):
mem_limit=None, user=None,
working_dir='/container/path'
)
+ mkdtemp_mock.assert_called_with(dir='/host/airflow',
+ prefix='airflowtmp', suffix='')
client_mock.create_host_config.assert_called_with(binds=['/host/path:/container/path',
'/mkdtemp:/tmp/airflow'],
network_mode='bridge')
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
> DockerOperator does not work with most recent version of Docker client
> ----------------------------------------------------------------------
>
> Key: AIRFLOW-1380
> URL: https://issues.apache.org/jira/browse/AIRFLOW-1380
> Project: Apache Airflow
> Issue Type: Bug
> Components: docker, operators
> Affects Versions: 1.9.0
> Environment: Tested on OS X, Python 3.5.2.
> Reporter: Benjamin Sims
> Assignee: Willem van Asperen
> Priority: Minor
> Fix For: 1.10.0, 2.0.0
>
> Original Estimate: 1h
> Remaining Estimate: 1h
>
> DockerOperator contains the following import:
> from docker import Client, tls
> https://github.com/apache/incubator-airflow/blob/master/airflow/operators/docker_operator.py#L21
> In the latest version of the docker Python client available in PyPy, 'Client' is no longer available and has been named 'APIClient'. The change was made here:
> https://github.com/docker/docker-py/commit/9daa320454ec0c19035a09b436a41e13c3fb71ad
> I will submit a PR with the relevant change.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)