You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by fo...@apache.org on 2018/05/24 08:25:28 UTC
incubator-airflow git commit:
[AIRFLOW-1057][AIRFLOW-1380][AIRFLOW-2362][2362] AIRFLOW Update
DockerOperator to new API
Repository: incubator-airflow
Updated Branches:
refs/heads/master ce9c7bbdf -> 49556917d
[AIRFLOW-1057][AIRFLOW-1380][AIRFLOW-2362][2362] AIRFLOW Update DockerOperator to new API
update import to docker's new API version >=2.0.0
changed dependency for docker package; now docker
rather than docker-py
updated test cases to align to new docker class
Closes #3407 from Noremac201/fixer
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/49556917
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/49556917
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/49556917
Branch: refs/heads/master
Commit: 49556917d9a3e9192d9043afd6cf3ca7939a2403
Parents: ce9c7bb
Author: Willem van Asperen <wi...@van.asperen.org>
Authored: Thu May 24 10:25:22 2018 +0200
Committer: Fokko Driesprong <fo...@godatadriven.com>
Committed: Thu May 24 10:25:22 2018 +0200
----------------------------------------------------------------------
airflow/hooks/docker_hook.py | 4 ++--
airflow/operators/docker_operator.py | 4 ++--
setup.py | 2 +-
tests/hooks/test_docker_hook.py | 3 +--
tests/operators/docker_operator.py | 36 +++++++++++++++----------------
5 files changed, 24 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/49556917/airflow/hooks/docker_hook.py
----------------------------------------------------------------------
diff --git a/airflow/hooks/docker_hook.py b/airflow/hooks/docker_hook.py
index 14a4a6e..5375c72 100644
--- a/airflow/hooks/docker_hook.py
+++ b/airflow/hooks/docker_hook.py
@@ -17,7 +17,7 @@
# specific language governing permissions and limitations
# under the License.
-from docker import Client
+from docker import APIClient
from docker.errors import APIError
from airflow.exceptions import AirflowException
@@ -60,7 +60,7 @@ class DockerHook(BaseHook, LoggingMixin):
self.__reauth = False if extra_options.get('reauth') == 'no' else True
def get_conn(self):
- client = Client(
+ client = APIClient(
base_url=self.__base_url,
version=self.__version,
tls=self.__tls
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/49556917/airflow/operators/docker_operator.py
----------------------------------------------------------------------
diff --git a/airflow/operators/docker_operator.py b/airflow/operators/docker_operator.py
index 9d535ff..421ce49 100644
--- a/airflow/operators/docker_operator.py
+++ b/airflow/operators/docker_operator.py
@@ -24,7 +24,7 @@ from airflow.exceptions import AirflowException
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
from airflow.utils.file import TemporaryDirectory
-from docker import Client, tls
+from docker import APIClient, tls
import ast
@@ -166,7 +166,7 @@ class DockerOperator(BaseOperator):
if self.docker_conn_id:
self.cli = self.get_hook().get_conn()
else:
- self.cli = Client(
+ self.cli = APIClient(
base_url=self.docker_url,
version=self.api_version,
tls=tls_config
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/49556917/setup.py
----------------------------------------------------------------------
diff --git a/setup.py b/setup.py
index 6d53ce5..dcd59af 100644
--- a/setup.py
+++ b/setup.py
@@ -136,7 +136,7 @@ doc = [
'sphinx-rtd-theme>=0.1.6',
'Sphinx-PyPI-upload>=0.2.1'
]
-docker = ['docker-py>=1.6.0']
+docker = ['docker>=2.0.0']
druid = ['pydruid>=0.4.1']
elasticsearch = [
'elasticsearch>=5.0.0,<6.0.0',
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/49556917/tests/hooks/test_docker_hook.py
----------------------------------------------------------------------
diff --git a/tests/hooks/test_docker_hook.py b/tests/hooks/test_docker_hook.py
index 5753aa5..b8a0132 100644
--- a/tests/hooks/test_docker_hook.py
+++ b/tests/hooks/test_docker_hook.py
@@ -26,7 +26,6 @@ from airflow.utils import db
try:
from airflow.hooks.docker_hook import DockerHook
- from docker import Client
except ImportError:
pass
@@ -39,7 +38,7 @@ except ImportError:
mock = None
-@mock.patch('airflow.hooks.docker_hook.Client', autospec=True)
+@mock.patch('airflow.hooks.docker_hook.APIClient', autospec=True)
class DockerHookTest(unittest.TestCase):
def setUp(self):
configuration.load_test_config()
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/49556917/tests/operators/docker_operator.py
----------------------------------------------------------------------
diff --git a/tests/operators/docker_operator.py b/tests/operators/docker_operator.py
index 1ebcbe9..78a920c 100644
--- a/tests/operators/docker_operator.py
+++ b/tests/operators/docker_operator.py
@@ -23,7 +23,7 @@ import logging
try:
from airflow.operators.docker_operator import DockerOperator
from airflow.hooks.docker_hook import DockerHook
- from docker import Client
+ from docker import APIClient
except ImportError:
pass
@@ -40,12 +40,12 @@ except ImportError:
class DockerOperatorTestCase(unittest.TestCase):
@mock.patch('airflow.utils.file.mkdtemp')
- @mock.patch('airflow.operators.docker_operator.Client')
+ @mock.patch('airflow.operators.docker_operator.APIClient')
def test_execute(self, client_class_mock, mkdtemp_mock):
host_config = mock.Mock()
mkdtemp_mock.return_value = '/mkdtemp'
- client_mock = mock.Mock(spec=Client)
+ client_mock = mock.Mock(spec=APIClient)
client_mock.create_container.return_value = {'Id': 'some_id'}
client_mock.create_host_config.return_value = host_config
client_mock.images.return_value = []
@@ -84,9 +84,9 @@ class DockerOperatorTestCase(unittest.TestCase):
client_mock.wait.assert_called_with('some_id')
@mock.patch('airflow.operators.docker_operator.tls.TLSConfig')
- @mock.patch('airflow.operators.docker_operator.Client')
+ @mock.patch('airflow.operators.docker_operator.APIClient')
def test_execute_tls(self, client_class_mock, tls_class_mock):
- client_mock = mock.Mock(spec=Client)
+ client_mock = mock.Mock(spec=APIClient)
client_mock.create_container.return_value = {'Id': 'some_id'}
client_mock.create_host_config.return_value = mock.Mock()
client_mock.images.return_value = []
@@ -104,15 +104,15 @@ class DockerOperatorTestCase(unittest.TestCase):
operator.execute(None)
tls_class_mock.assert_called_with(assert_hostname=None, ca_cert='ca.pem',
- client_cert=('cert.pem', 'key.pem'), ssl_version=None,
- verify=True)
+ client_cert=('cert.pem', 'key.pem'),
+ ssl_version=None, verify=True)
- client_class_mock.assert_called_with(base_url='https://127.0.0.1:2376', tls=tls_mock,
- version=None)
+ client_class_mock.assert_called_with(base_url='https://127.0.0.1:2376',
+ tls=tls_mock, version=None)
- @mock.patch('airflow.operators.docker_operator.Client')
+ @mock.patch('airflow.operators.docker_operator.APIClient')
def test_execute_unicode_logs(self, client_class_mock):
- client_mock = mock.Mock(spec=Client)
+ client_mock = mock.Mock(spec=APIClient)
client_mock.create_container.return_value = {'Id': 'some_id'}
client_mock.create_host_config.return_value = mock.Mock()
client_mock.images.return_value = []
@@ -132,9 +132,9 @@ class DockerOperatorTestCase(unittest.TestCase):
logging.raiseExceptions = originalRaiseExceptions
print_exception_mock.assert_not_called()
- @mock.patch('airflow.operators.docker_operator.Client')
+ @mock.patch('airflow.operators.docker_operator.APIClient')
def test_execute_container_fails(self, client_class_mock):
- client_mock = mock.Mock(spec=Client)
+ client_mock = mock.Mock(spec=APIClient)
client_mock.create_container.return_value = {'Id': 'some_id'}
client_mock.create_host_config.return_value = mock.Mock()
client_mock.images.return_value = []
@@ -150,7 +150,7 @@ class DockerOperatorTestCase(unittest.TestCase):
operator.execute(None)
def test_on_kill(self):
- client_mock = mock.Mock(spec=Client)
+ client_mock = mock.Mock(spec=APIClient)
operator = DockerOperator(image='ubuntu', owner='unittest', task_id='unittest')
operator.cli = client_mock
@@ -160,10 +160,10 @@ class DockerOperatorTestCase(unittest.TestCase):
client_mock.stop.assert_called_with('some_id')
- @mock.patch('airflow.operators.docker_operator.Client')
+ @mock.patch('airflow.operators.docker_operator.APIClient')
def test_execute_no_docker_conn_id_no_hook(self, operator_client_mock):
# Mock out a Docker client, so operations don't raise errors
- client_mock = mock.Mock(name='DockerOperator.Client mock', spec=Client)
+ client_mock = mock.Mock(name='DockerOperator.APIClient mock', spec=APIClient)
client_mock.images.return_value = []
client_mock.create_container.return_value = {'Id': 'some_id'}
client_mock.logs.return_value = []
@@ -194,11 +194,11 @@ class DockerOperatorTestCase(unittest.TestCase):
)
@mock.patch('airflow.operators.docker_operator.DockerHook')
- @mock.patch('airflow.operators.docker_operator.Client')
+ @mock.patch('airflow.operators.docker_operator.APIClient')
def test_execute_with_docker_conn_id_use_hook(self, operator_client_mock,
operator_docker_hook):
# Mock out a Docker client, so operations don't raise errors
- client_mock = mock.Mock(name='DockerOperator.Client mock', spec=Client)
+ client_mock = mock.Mock(name='DockerOperator.APIClient mock', spec=APIClient)
client_mock.images.return_value = []
client_mock.create_container.return_value = {'Id': 'some_id'}
client_mock.logs.return_value = []