You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by "Harish (Jira)" <ji...@apache.org> on 2020/01/02 19:12:00 UTC
[jira] [Updated] (AIRFLOW-6431) docker operator and xcom
[ https://issues.apache.org/jira/browse/AIRFLOW-6431?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Harish updated AIRFLOW-6431:
----------------------------
Description:
airflow with celery executor.
a DockerOprator task that runs [the hello-world docker|https://hub.docker.com/_/hello-world]
{code:python}
from airflow import DAG
from airflow.operators.docker_operator import DockerOperator
args = {
'owner': 'airflow',
'start_date': airflow.utils.dates.days_ago(2),
}
dag = DAG('example_xcom', schedule_interval="@once", default_args=args)
d = DockerOperator(
task_id='hello-world',
image='hello-world',
# auto_remove=True,
do_xcom_push=True,
xcom_all=True,
tty=True,
dag=dag
)
{code}
3 issues:
# the task log does not show messages printed by the docker to the standard output.
# no xcom is created.
# an exception is thrown if run with auto_remove=True
{code:python}
[2020-01-02 19:01:54,438] {docker_operator.py:214} INFO - Starting docker container from image hello-world
[2020-01-02 19:01:54,799] {taskinstance.py:1078} ERROR - 404 Client Error: Not Found ("No such container: 98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e")
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 261, in _raise_for_status
response.raise_for_status()
File "/usr/local/lib/python3.6/site-packages/requests/models.py", line 940, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 404 Client Error: Not Found for url: http+docker://localhost/v1.35/containers/98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e/json
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/airflow/airflow/models/taskinstance.py", line 945, in _run_raw_task
result = task_copy.execute(context=context)
File "/opt/airflow/airflow/operators/docker_operator.py", line 283, in execute
self._run_image()
File "/opt/airflow/airflow/operators/docker_operator.py", line 256, in _run_image
if self.xcom_all else line.encode('utf-8')
File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped
return f(self, resource_id, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/docker/api/container.py", line 852, in logs
output = self._get_result(container, stream, res)
File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 451, in _get_result
return self._get_result_tty(stream, res, self._check_is_tty(container))
File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped
return f(self, resource_id, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 447, in _check_is_tty
cont = self.inspect_container(container)
File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped
return f(self, resource_id, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/docker/api/container.py", line 755, in inspect_container
self._get(self._url("/containers/{0}/json", container)), True
File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 267, in _result
self._raise_for_status(response)
File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 263, in _raise_for_status
raise create_api_error_from_http_exception(e)
File "/usr/local/lib/python3.6/site-packages/docker/errors.py", line 31, in create_api_error_from_http_exception
raise cls(e, response=response, explanation=explanation)
docker.errors.NotFound: 404 Client Error: Not Found ("No such container: 98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e")
[2020-01-02 19:01:54,800] {taskinstance.py:1109} INFO - Marking task as FAILED.
[2020-01-02 19:01:59,349] {logging_mixin.py:89} INFO - [2020-01-02 19:01:59,349] {local_task_job.py:98} INFO - Task exited with return code 1
{code}
was:
airflow with celery executor.
a DockerOprator task that runs [the hello-world docker|https://hub.docker.com/_/hello-world]
{code:python}
from airflow import DAG
from airflow.operators.docker_operator import DockerOperatorargs = {
'owner': 'airflow',
'start_date': airflow.utils.dates.days_ago(2),
}
dag = DAG('example_xcom', schedule_interval="@once", default_args=args)d = DockerOperator(
task_id='hello-world',
image='hello-world',
# auto_remove=True,
do_xcom_push=True,
xcom_all=True,
tty=True,
dag=dag
)
{code}
3 issues:
# the task log does not show messages printed by the docker to the standard output.
# no xcom is created.
# an exception is thrown if run with auto_remove=True
{code:python}
[2020-01-02 19:01:54,438] {docker_operator.py:214} INFO - Starting docker container from image hello-world
[2020-01-02 19:01:54,799] {taskinstance.py:1078} ERROR - 404 Client Error: Not Found ("No such container: 98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e")
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 261, in _raise_for_status
response.raise_for_status()
File "/usr/local/lib/python3.6/site-packages/requests/models.py", line 940, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 404 Client Error: Not Found for url: http+docker://localhost/v1.35/containers/98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e/json
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/airflow/airflow/models/taskinstance.py", line 945, in _run_raw_task
result = task_copy.execute(context=context)
File "/opt/airflow/airflow/operators/docker_operator.py", line 283, in execute
self._run_image()
File "/opt/airflow/airflow/operators/docker_operator.py", line 256, in _run_image
if self.xcom_all else line.encode('utf-8')
File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped
return f(self, resource_id, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/docker/api/container.py", line 852, in logs
output = self._get_result(container, stream, res)
File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 451, in _get_result
return self._get_result_tty(stream, res, self._check_is_tty(container))
File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped
return f(self, resource_id, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 447, in _check_is_tty
cont = self.inspect_container(container)
File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped
return f(self, resource_id, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/docker/api/container.py", line 755, in inspect_container
self._get(self._url("/containers/{0}/json", container)), True
File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 267, in _result
self._raise_for_status(response)
File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 263, in _raise_for_status
raise create_api_error_from_http_exception(e)
File "/usr/local/lib/python3.6/site-packages/docker/errors.py", line 31, in create_api_error_from_http_exception
raise cls(e, response=response, explanation=explanation)
docker.errors.NotFound: 404 Client Error: Not Found ("No such container: 98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e")
[2020-01-02 19:01:54,800] {taskinstance.py:1109} INFO - Marking task as FAILED.
[2020-01-02 19:01:59,349] {logging_mixin.py:89} INFO - [2020-01-02 19:01:59,349] {local_task_job.py:98} INFO - Task exited with return code 1
{code}
> docker operator and xcom
> ------------------------
>
> Key: AIRFLOW-6431
> URL: https://issues.apache.org/jira/browse/AIRFLOW-6431
> Project: Apache Airflow
> Issue Type: Bug
> Components: operators, xcom
> Affects Versions: 2.0.0
> Reporter: Harish
> Priority: Major
>
> airflow with celery executor.
> a DockerOprator task that runs [the hello-world docker|https://hub.docker.com/_/hello-world]
> {code:python}
> from airflow import DAG
> from airflow.operators.docker_operator import DockerOperator
> args = {
> 'owner': 'airflow',
> 'start_date': airflow.utils.dates.days_ago(2),
> }
> dag = DAG('example_xcom', schedule_interval="@once", default_args=args)
> d = DockerOperator(
> task_id='hello-world',
> image='hello-world',
> # auto_remove=True,
> do_xcom_push=True,
> xcom_all=True,
> tty=True,
> dag=dag
> )
> {code}
>
> 3 issues:
> # the task log does not show messages printed by the docker to the standard output.
> # no xcom is created.
> # an exception is thrown if run with auto_remove=True
> {code:python}
> [2020-01-02 19:01:54,438] {docker_operator.py:214} INFO - Starting docker container from image hello-world
> [2020-01-02 19:01:54,799] {taskinstance.py:1078} ERROR - 404 Client Error: Not Found ("No such container: 98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e")
> Traceback (most recent call last):
> File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 261, in _raise_for_status
> response.raise_for_status()
> File "/usr/local/lib/python3.6/site-packages/requests/models.py", line 940, in raise_for_status
> raise HTTPError(http_error_msg, response=self)
> requests.exceptions.HTTPError: 404 Client Error: Not Found for url: http+docker://localhost/v1.35/containers/98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e/json
> During handling of the above exception, another exception occurred:
> Traceback (most recent call last):
> File "/opt/airflow/airflow/models/taskinstance.py", line 945, in _run_raw_task
> result = task_copy.execute(context=context)
> File "/opt/airflow/airflow/operators/docker_operator.py", line 283, in execute
> self._run_image()
> File "/opt/airflow/airflow/operators/docker_operator.py", line 256, in _run_image
> if self.xcom_all else line.encode('utf-8')
> File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped
> return f(self, resource_id, *args, **kwargs)
> File "/usr/local/lib/python3.6/site-packages/docker/api/container.py", line 852, in logs
> output = self._get_result(container, stream, res)
> File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 451, in _get_result
> return self._get_result_tty(stream, res, self._check_is_tty(container))
> File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped
> return f(self, resource_id, *args, **kwargs)
> File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 447, in _check_is_tty
> cont = self.inspect_container(container)
> File "/usr/local/lib/python3.6/site-packages/docker/utils/decorators.py", line 19, in wrapped
> return f(self, resource_id, *args, **kwargs)
> File "/usr/local/lib/python3.6/site-packages/docker/api/container.py", line 755, in inspect_container
> self._get(self._url("/containers/{0}/json", container)), True
> File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 267, in _result
> self._raise_for_status(response)
> File "/usr/local/lib/python3.6/site-packages/docker/api/client.py", line 263, in _raise_for_status
> raise create_api_error_from_http_exception(e)
> File "/usr/local/lib/python3.6/site-packages/docker/errors.py", line 31, in create_api_error_from_http_exception
> raise cls(e, response=response, explanation=explanation)
> docker.errors.NotFound: 404 Client Error: Not Found ("No such container: 98456def71fc1f5f7a3d616f4b3a095d89bf64aafcca8b4fd0df1ff229083d9e")
> [2020-01-02 19:01:54,800] {taskinstance.py:1109} INFO - Marking task as FAILED.
> [2020-01-02 19:01:59,349] {logging_mixin.py:89} INFO - [2020-01-02 19:01:59,349] {local_task_job.py:98} INFO - Task exited with return code 1
> {code}
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)