You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by sa...@apache.org on 2016/11/17 08:15:40 UTC
incubator-airflow git commit: [AIRFLOW-565] Fixes DockerOperator on
Python3.x
Repository: incubator-airflow
Updated Branches:
refs/heads/master d91082124 -> e1bc51691
[AIRFLOW-565] Fixes DockerOperator on Python3.x
The issue is that `self.cli.pull()` returns
`bytes()`, and not a string. Then,
when we try to pass that to `json.loads()`, it
raises an exception.
The fix is to convert the bytes to a string by
decoding it as "utf-8". We're
hardcoding the encoding because, by the JSON
schema, a JSON should encoded in
UTF-8, UTF-16 or UTF-32. Considering we're only
pulling images from Docker
servers, we can be relatively safe that they'll
behave correctly.
Closes #1832 from vitorbaptista/bug/fixes-
AIRFLOW-565
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/e1bc5169
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/e1bc5169
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/e1bc5169
Branch: refs/heads/master
Commit: e1bc5169181e7fcf6b8068f7da309c35b87a05b7
Parents: d910821
Author: Vitor Baptista <vi...@vitorbaptista.com>
Authored: Thu Nov 17 00:15:30 2016 -0800
Committer: Siddharth Anand <si...@yahoo.com>
Committed: Thu Nov 17 00:15:30 2016 -0800
----------------------------------------------------------------------
airflow/operators/docker_operator.py | 2 +-
tests/operators/docker_operator.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/e1bc5169/airflow/operators/docker_operator.py
----------------------------------------------------------------------
diff --git a/airflow/operators/docker_operator.py b/airflow/operators/docker_operator.py
index d17d154..c75e776 100644
--- a/airflow/operators/docker_operator.py
+++ b/airflow/operators/docker_operator.py
@@ -152,7 +152,7 @@ class DockerOperator(BaseOperator):
if self.force_pull or len(self.cli.images(name=image)) == 0:
logging.info('Pulling docker image ' + image)
for l in self.cli.pull(image, stream=True):
- output = json.loads(l)
+ output = json.loads(l.decode('utf-8'))
logging.info("{}".format(output['status']))
cpu_shares = int(round(self.cpus * 1024))
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/e1bc5169/tests/operators/docker_operator.py
----------------------------------------------------------------------
diff --git a/tests/operators/docker_operator.py b/tests/operators/docker_operator.py
index 5a405ad..cdfae94 100644
--- a/tests/operators/docker_operator.py
+++ b/tests/operators/docker_operator.py
@@ -44,7 +44,7 @@ class DockerOperatorTestCase(unittest.TestCase):
client_mock.create_host_config.return_value = host_config
client_mock.images.return_value = []
client_mock.logs.return_value = ['container log']
- client_mock.pull.return_value = ['{"status":"pull log"}']
+ client_mock.pull.return_value = [b'{"status":"pull log"}']
client_mock.wait.return_value = 0
client_class_mock.return_value = client_mock