You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by GitBox <gi...@apache.org> on 2020/10/29 14:40:19 UTC
[GitHub] [airflow] nirousseau commented on issue #9860: Kubernetes Executor Config Volumes Break Airflow UI
nirousseau commented on issue #9860:
URL: https://github.com/apache/airflow/issues/9860#issuecomment-718797666
Hello,
I am also facing the issue described above (UI Breaks).
**Airflow 1.10.12** with KubernetesExecutor.
here is my dag :
```python
# -*- coding: utf-8 -*-
"""
### Tutorial Documentation
Documentation that goes along with the Airflow tutorial located
[here](https://airflow.apache.org/tutorial.html)
"""
# [START tutorial]
# [START import_module]
from datetime import timedelta
from helpers.k8s_executor import get_config as executor_config
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from airflow.utils.dates import days_ago
# [END import_module]
# [START default_args]
# These args will get passed on to each operator
# You can override them on a per-task basis during operator initialization
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': days_ago(2),
'email': ['airflow@example.com'],
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
# 'queue': 'bash_queue',
# 'pool': 'backfill',
# 'priority_weight': 10,
# 'end_date': datetime(2016, 1, 1),
# 'wait_for_downstream': False,
# 'dag': dag,
# 'sla': timedelta(hours=2),
# 'execution_timeout': timedelta(seconds=300),
# 'on_failure_callback': some_function,
# 'on_success_callback': some_other_function,
# 'on_retry_callback': another_function,
# 'sla_miss_callback': yet_another_function,
# 'trigger_rule': 'all_success'
}
# [END default_args]
# [START instantiate_dag]
with DAG(
'k8s_tutorial',
default_args=default_args,
description='A simple tutorial DAG on K8S',
schedule_interval=timedelta(days=1),
) as dag:
# [END instantiate_dag]
# t1, t2 and t3 are examples of tasks created by instantiating operators
# [START basic_task]
t1 = BashOperator(
task_id='print_date',
bash_command='date',
executor_config=executor_config()
)
t2 = BashOperator(
task_id='sleep',
depends_on_past=False,
bash_command='sleep 300',
retries=3,
executor_config=executor_config()
)
# [END basic_task]
# [START documentation]
dag.doc_md = __doc__
t1.doc_md = """\
#### Task Documentation
You can document your task using the attributes `doc_md` (markdown),
`doc` (plain text), `doc_rst`, `doc_json`, `doc_yaml` which gets
rendered in the UI's Task Instance Details page.
"""
# [END documentation]
t1 >> t2
# [END tutorial]
```
here is the helper included that triggers the UI crash :
```python
def get_config():
"""
Override Executor Configuration for K8S
:return:
"""
from kubernetes.client import models as k8s
return {
"KubernetesExecutor": {
"request_cpu": "200m",
"limit_cpu": "200m",
"request_memory": "128Mi",
"limit_memory": "128Mi",
"labels": {
"app": "airflow",
"airflow-postgresql-client": "true"
},
"security_context": k8s.V1PodSecurityContext(
run_as_user=1000,
run_as_group=1000
),
"init_containers": [
k8s.V1Container(
name="git-sync-clone",
image="my_private_registry/k8s/images/git:latest",
command=[
"/scripts/git-clone.sh"
],
args=[
"MY_GIT_REPO",
"master",
"/git",
"MY_GIT_HOST",
"443",
"id_rsa"
],
resources=k8s.V1ResourceRequirements(
requests={"cpu": "100m", "memory": "64Mi"},
limits={"cpu": "100m", "memory": "64Mi"}
),
security_context=k8s.V1PodSecurityContext(
run_as_user=1000,
run_as_group=1000
),
volume_mounts=[
k8s.V1VolumeMount(
mount_path="/git",
name="airflow-dags",
read_only=False
),
k8s.V1VolumeMount(
mount_path="/scripts",
name="git-clone",
read_only=True
)
]
)
],
"volumes": [
k8s.V1Volume(
name="git-clone",
config_map=k8s.V1ConfigMapVolumeSource(
default_mode=0o777,
name="sandbox-airflow-scripts-git"
)
),
k8s.V1Volume(
name="airflow-dags",
empty_dir=k8s.V1EmptyDirVolumeSource()
),
k8s.V1Volume(
name="airflow-logs",
empty_dir=k8s.V1EmptyDirVolumeSource()
)
]
}
}
```
Crash traceback :
```
____/ ( ( ) ) \___
/( ( ( ) _ )) ) )\
(( ( )( ) ) ( ) )
((/ ( _( ) ( _) ) ( () ) )
( ( ( (_) (( ( ) .((_ ) . )_
( ( ) ( ( ) ) ) . ) ( )
( ( ( ( ) ( _ ( _) ). ) . ) ) ( )
( ( ( ) ( ) ( )) ) _)( ) ) )
( ( ( \ ) ( (_ ( ) ( ) ) ) ) )) ( )
( ( ( ( (_ ( ) ( _ ) ) ( ) ) )
( ( ( ( ( ) (_ ) ) ) _) ) _( ( )
(( ( )( ( _ ) _) _(_ ( (_ )
(_((__(_(__(( ( ( | ) ) ) )_))__))_)___)
((__) \\||lll|l||/// \_))
( /(/ ( ) ) )\ )
( ( ( ( | | ) ) )\ )
( /(| / ( )) ) ) )) )
( ( ((((_(|)_))))) )
( ||\(|(|)|/|| )
( |(||(||)|||| )
( //|/l|||)|\\ \ )
(/ / // /|//||||\\ \ \ \ _)
-------------------------------------------------------------------------------
Node: sandbox-airflow-web-69c87579cf-p94wg
-------------------------------------------------------------------------------
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/usr/local/lib/python3.7/site-packages/flask/_compat.py", line 39, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/usr/local/lib/python3.7/site-packages/flask_admin/base.py", line 69, in inner
return self._run_view(f, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/flask_admin/base.py", line 368, in _run_view
return fn(self, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/flask_login/utils.py", line 258, in decorated_view
return func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/airflow/www/utils.py", line 384, in view_func
return f(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/airflow/www/utils.py", line 290, in wrapper
return f(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/airflow/utils/db.py", line 74, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/airflow/www/views.py", line 1779, in graph
show_external_logs=bool(external_logs))
File "/usr/local/lib/python3.7/site-packages/airflow/www/views.py", line 405, in render
return super(AirflowViewMixin, self).render(template, **kwargs)
File "/usr/local/lib/python3.7/site-packages/flask_admin/base.py", line 308, in render
return render_template(template, **kwargs)
File "/usr/local/lib/python3.7/site-packages/flask/templating.py", line 140, in render_template
ctx.app,
File "/usr/local/lib/python3.7/site-packages/flask/templating.py", line 120, in _render
rv = template.render(context)
File "/usr/local/lib/python3.7/site-packages/jinja2/environment.py", line 1090, in render
self.environment.handle_exception()
File "/usr/local/lib/python3.7/site-packages/jinja2/environment.py", line 832, in handle_exception
reraise(*rewrite_traceback_stack(source=source))
File "/usr/local/lib/python3.7/site-packages/jinja2/_compat.py", line 28, in reraise
raise value.with_traceback(tb)
File "/usr/local/lib/python3.7/site-packages/airflow/www/templates/airflow/graph.html", line 22, in top-level template code
{% import 'admin/static.html' as admin_static with context %}
File "/usr/local/lib/python3.7/site-packages/airflow/www/templates/airflow/dag.html", line 21, in top-level template code
{% import 'admin/lib.html' as lib with context %}
File "/usr/local/lib/python3.7/site-packages/airflow/www/templates/airflow/master.html", line 20, in top-level template code
{% extends "admin/master.html" %}
File "/usr/local/lib/python3.7/site-packages/airflow/www/templates/admin/master.html", line 20, in top-level template code
{% extends 'admin/base.html' %}
File "/usr/local/lib/python3.7/site-packages/flask_admin/templates/bootstrap3/admin/base.html", line 95, in top-level template code
{% block tail %}
File "/usr/local/lib/python3.7/site-packages/airflow/www/templates/airflow/graph.html", line 115, in block "tail"
var task_instances = {{ task_instances|tojson }};
File "/usr/local/lib/python3.7/site-packages/flask/json/__init__.py", line 376, in tojson_filter
return Markup(htmlsafe_dumps(obj, **kwargs))
File "/usr/local/lib/python3.7/site-packages/flask/json/__init__.py", line 290, in htmlsafe_dumps
dumps(obj, **kwargs)
File "/usr/local/lib/python3.7/site-packages/flask/json/__init__.py", line 211, in dumps
rv = _json.dumps(obj, **kwargs)
File "/usr/local/lib/python3.7/json/__init__.py", line 238, in dumps
**kw).encode(obj)
File "/usr/local/lib/python3.7/json/encoder.py", line 199, in encode
chunks = self.iterencode(o, _one_shot=True)
File "/usr/local/lib/python3.7/json/encoder.py", line 257, in iterencode
return _iterencode(o, 0)
File "/usr/local/lib/python3.7/site-packages/flask/json/__init__.py", line 100, in default
return _json.JSONEncoder.default(self, o)
File "/usr/local/lib/python3.7/json/encoder.py", line 179, in default
raise TypeError(f'Object of type {o.__class__.__name__} '
TypeError: Object of type V1Container is not JSON serializable
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org