You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bo...@apache.org on 2018/01/19 17:54:32 UTC
incubator-airflow git commit: [AIRFLOW-1755] Allow mount below root
Repository: incubator-airflow
Updated Branches:
refs/heads/master c3c4a8fdc -> 1e36b37b6
[AIRFLOW-1755] Allow mount below root
This enables Airflow and Celery Flower to live
below root. Draws on the work of Geatan Semet
(@Stibbons).
This closes #2723 and closes #2818
Closes #2952 from bolkedebruin/AIRFLOW-1755
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/1e36b37b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/1e36b37b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/1e36b37b
Branch: refs/heads/master
Commit: 1e36b37b68ab354d1d7d1d1d3abd151ce2a7cac7
Parents: c3c4a8f
Author: Bolke de Bruin <bo...@xs4all.nl>
Authored: Fri Jan 19 18:54:26 2018 +0100
Committer: Bolke de Bruin <bo...@xs4all.nl>
Committed: Fri Jan 19 18:54:26 2018 +0100
----------------------------------------------------------------------
airflow/bin/cli.py | 18 +++++--
airflow/config_templates/default_airflow.cfg | 8 +++
.../templates/metastore_browser/base.html | 10 ++--
airflow/www/app.py | 20 +++++--
airflow/www/templates/airflow/dag_details.html | 8 +--
airflow/www/templates/airflow/dags.html | 18 +++----
airflow/www/templates/airflow/graph.html | 8 +--
airflow/www/templates/airflow/list_dags.html | 4 +-
airflow/www/templates/airflow/nvd3.html | 8 +--
airflow/www/views.py | 3 ++
docs/integration.rst | 57 ++++++++++++++++++++
setup.py | 1 +
tests/www/test_views.py | 22 ++++++++
13 files changed, 150 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/bin/cli.py
----------------------------------------------------------------------
diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py
index e98838d..b032729 100755
--- a/airflow/bin/cli.py
+++ b/airflow/bin/cli.py
@@ -1054,6 +1054,10 @@ def flower(args):
if args.broker_api:
api = '--broker_api=' + args.broker_api
+ url_prefix = ''
+ if args.url_prefix:
+ url_prefix = '--url-prefix=' + args.url_prefix
+
flower_conf = ''
if args.flower_conf:
flower_conf = '--conf=' + args.flower_conf
@@ -1070,7 +1074,8 @@ def flower(args):
)
with ctx:
- os.execvp("flower", ['flower', '-b', broka, address, port, api, flower_conf])
+ os.execvp("flower", ['flower', '-b',
+ broka, address, port, api, flower_conf, url_prefix])
stdout.close()
stderr.close()
@@ -1078,7 +1083,8 @@ def flower(args):
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
- os.execvp("flower", ['flower', '-b', broka, address, port, api, flower_conf])
+ os.execvp("flower", ['flower', '-b',
+ broka, address, port, api, flower_conf, url_prefix])
def kerberos(args): # noqa
@@ -1410,6 +1416,10 @@ class CLIFactory(object):
'flower_conf': Arg(
("-fc", "--flower_conf"),
help="Configuration file for flower"),
+ 'flower_url_prefix': Arg(
+ ("-u", "--url_prefix"),
+ default=conf.get('celery', 'FLOWER_URL_PREFIX'),
+ help="URL prefix for Flower"),
'task_params': Arg(
("-tp", "--task_params"),
help="Sends a JSON params dict to the task"),
@@ -1584,8 +1594,8 @@ class CLIFactory(object):
}, {
'func': flower,
'help': "Start a Celery Flower",
- 'args': ('flower_hostname', 'flower_port', 'flower_conf', 'broker_api',
- 'pid', 'daemon', 'stdout', 'stderr', 'log_file'),
+ 'args': ('flower_hostname', 'flower_port', 'flower_conf', 'flower_url_prefix',
+ 'broker_api', 'pid', 'daemon', 'stdout', 'stderr', 'log_file'),
}, {
'func': version,
'help': "Show the version",
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/config_templates/default_airflow.cfg
----------------------------------------------------------------------
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index d0dfb72..59ff740 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -155,6 +155,10 @@ killed_task_cleanup_time = 60
# database directly, while the json_client will use the api running on the
# webserver
api_client = airflow.api.client.local_client
+
+# If you set web_server_url_prefix, do NOT forget to append it here, ex:
+# endpoint_url = http://localhost:8080/myroot
+# So api will look like: http://localhost:8080/myroot/api/experimental/...
endpoint_url = http://localhost:8080
[api]
@@ -312,6 +316,10 @@ result_backend = db+mysql://airflow:airflow@localhost:3306/airflow
# it `airflow flower`. This defines the IP that Celery Flower runs on
flower_host = 0.0.0.0
+# The root URL for Flower
+# Ex: flower_url_prefix = /flower
+flower_url_prefix =
+
# This defines the port that Celery Flower runs on
flower_port = 5555
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/contrib/plugins/metastore_browser/templates/metastore_browser/base.html
----------------------------------------------------------------------
diff --git a/airflow/contrib/plugins/metastore_browser/templates/metastore_browser/base.html b/airflow/contrib/plugins/metastore_browser/templates/metastore_browser/base.html
index ba1e7e2..26675bb 100644
--- a/airflow/contrib/plugins/metastore_browser/templates/metastore_browser/base.html
+++ b/airflow/contrib/plugins/metastore_browser/templates/metastore_browser/base.html
@@ -1,13 +1,13 @@
-{#
+{#
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -31,14 +31,14 @@
{% block head %}
{{ super() }}
<link rel="stylesheet" type="text/css" href="{{ url_for("static", filename="dataTables.bootstrap.css") }}">
-<link href="/admin/static/vendor/select2/select2.css" rel="stylesheet">
+<link href="{{ url_for('static', filename='select2.css') }}" rel="stylesheet">
{% endblock %}
{% block tail %}
{{ super() }}
<script src="{{ url_for('static', filename='jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='dataTables.bootstrap.js') }}"></script>
-<script src="/admin/static/vendor/select2/select2.min.js" type="text/javascript"></script>
+<script src="{{ url_for('static', filename='select2.min.js') }}" type="text/javascript"></script>
<script>
// Filling up the table selector
url = "{{ url_for('.objects') }}";
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/www/app.py
----------------------------------------------------------------------
diff --git a/airflow/www/app.py b/airflow/www/app.py
index 0b71c17..d46935b 100644
--- a/airflow/www/app.py
+++ b/airflow/www/app.py
@@ -19,7 +19,8 @@ from flask import Flask
from flask_admin import Admin, base
from flask_caching import Cache
from flask_wtf.csrf import CSRFProtect
-csrf = CSRFProtect()
+from six.moves.urllib.parse import urlparse
+from werkzeug.wsgi import DispatcherMiddleware
import airflow
from airflow import configuration as conf
@@ -32,6 +33,8 @@ from airflow import jobs
from airflow import settings
from airflow import configuration
+csrf = CSRFProtect()
+
def create_app(config=None, testing=False):
app = Flask(__name__)
@@ -154,11 +157,22 @@ def create_app(config=None, testing=False):
return app
+
app = None
-def cached_app(config=None):
+def root_app(env, resp):
+ resp(b'404 Not Found', [(b'Content-Type', b'text/plain')])
+ return [b'Apache Airflow is not at this location']
+
+
+def cached_app(config=None, testing=False):
global app
if not app:
- app = create_app(config)
+ base_url = urlparse(configuration.get('webserver', 'base_url'))[2]
+ if not base_url or base_url == '/':
+ base_url = ""
+
+ app = create_app(config, testing)
+ app = DispatcherMiddleware(root_app, {base_url: app})
return app
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/www/templates/airflow/dag_details.html
----------------------------------------------------------------------
diff --git a/airflow/www/templates/airflow/dag_details.html b/airflow/www/templates/airflow/dag_details.html
index 716932a..4f9f2eb 100644
--- a/airflow/www/templates/airflow/dag_details.html
+++ b/airflow/www/templates/airflow/dag_details.html
@@ -1,13 +1,13 @@
-{#
+{#
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -29,7 +29,7 @@
<a
class="btn"
style="border: none; background-color:{{ State.color(state)}}; color: {{ State.color_fg(state) }};"
- href="/admin/taskinstance/?flt0_dag_id_equals={{ dag.dag_id }}&flt2_state_equals={{ state }}">
+ href="{{ url_for('taskinstance.index_view') }}?flt0_dag_id_equals={{ dag.dag_id }}&flt2_state_equals={{ state }}">
{{ state }} <span class="badge">{{ count }}</span>
</a>
{% endfor %}
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/www/templates/airflow/dags.html
----------------------------------------------------------------------
diff --git a/airflow/www/templates/airflow/dags.html b/airflow/www/templates/airflow/dags.html
index eeef790..6b051c3 100644
--- a/airflow/www/templates/airflow/dags.html
+++ b/airflow/www/templates/airflow/dags.html
@@ -69,7 +69,7 @@
<!-- Column 1: Edit dag -->
<td class="text-center" style="width:10px;">
{% if dag_id in orm_dags %}
- <a href="/admin/dagmodel/edit/?id={{ dag_id }}" title="Info">
+ <a href="{{ url_for('dagmodel.edit_view') }}?id={{ dag_id }}" title="Info">
<span class="glyphicon glyphicon-edit" aria-hidden="true"></span>
</a>
{% endif %}
@@ -100,7 +100,7 @@
<!-- Column 4: Dag Schedule -->
<td>
{% if dag_id in webserver_dags %}
- <a class="label label-default schedule {{ dag.dag_id }}" href="/admin/dagrun/?flt2_dag_id_equals={{ dag.dag_id }}">
+ <a class="label label-default schedule {{ dag.dag_id }}" href="{{ url_for('dagrun.index_view') }}?flt2_dag_id_equals={{ dag.dag_id }}">
{{ dag.schedule_interval }}
</a>
{% endif %}
@@ -180,7 +180,7 @@
</a>
<!-- Logs -->
- <a href="/admin/log/?sort=1&desc=1&flt1_dag_id_equals={{ dag.dag_id }}">
+ <a href="{{ url_for('log.index_view') }}?sort=1&desc=1&flt1_dag_id_equals={{ dag.dag_id }}">
<span class="glyphicon glyphicon-align-justify" aria-hidden="true" data-original-title="Logs"></span>
</a>
{% endif %}
@@ -209,9 +209,9 @@
</div>
{% if not hide_paused %}
- <a href="/admin/?showPaused=False">Hide Paused DAGs</a>
+ <a href="{{ url_for('admin.index') }}?showPaused=False">Hide Paused DAGs</a>
{% else %}
- <a href="/admin/?showPaused=True">Show Paused DAGs</a>
+ <a href="{{ url_for('admin.index') }}?showPaused=True">Show Paused DAGs</a>
{% endif %}
</div>
{% endblock %}
@@ -224,8 +224,8 @@
<script src="{{ url_for('static', filename='bootstrap3-typeahead.min.js') }}"></script>
<script>
- const DAGS_INDEX = {{ url_for('admin.index') }}
- const ENTER_KEY_CODE = 13
+ const DAGS_INDEX = "{{ url_for('admin.index') }}";
+ const ENTER_KEY_CODE = 13;
$('#dag_query').on('keypress', function (e) {
// check for key press on ENTER (key code 13) to trigger the search
@@ -353,7 +353,7 @@
})
.on('click', function(d, i) {
if (d.count > 0)
- window.location = "/admin/dagrun/?flt1_dag_id_equals=" + d.dag_id + "&flt2_state_equals=" + d.state;
+ window.location = "{{ url_for('dagrun.index_view') }}?flt1_dag_id_equals=" + d.dag_id + "&flt2_state_equals=" + d.state;
})
.on('mouseover', function(d, i) {
if (d.count > 0) {
@@ -432,7 +432,7 @@
})
.on('click', function(d, i) {
if (d.count > 0)
- window.location = "/admin/taskinstance/?flt1_dag_id_equals=" + d.dag_id + "&flt2_state_equals=" + d.state;
+ window.location = "{{ url_for('taskinstance.index_view') }}?flt1_dag_id_equals=" + d.dag_id + "&flt2_state_equals=" + d.state;
})
.on('mouseover', function(d, i) {
if (d.count > 0) {
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/www/templates/airflow/graph.html
----------------------------------------------------------------------
diff --git a/airflow/www/templates/airflow/graph.html b/airflow/www/templates/airflow/graph.html
index 24fc508..e17c89b 100644
--- a/airflow/www/templates/airflow/graph.html
+++ b/airflow/www/templates/airflow/graph.html
@@ -1,13 +1,13 @@
-{#
+{#
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -350,7 +350,7 @@
$("#loading").css("display", "block");
$("div#svg_container").css("opacity", "0.2");
$.get(
- "/admin/airflow/object/task_instances",
+ "{{ url_for('airflow.task_instances') }}",
{dag_id : "{{ dag.dag_id }}", execution_date : "{{ execution_date }}"})
.done(
function(task_instances) {
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/www/templates/airflow/list_dags.html
----------------------------------------------------------------------
diff --git a/airflow/www/templates/airflow/list_dags.html b/airflow/www/templates/airflow/list_dags.html
index 9ace2fd..e8533d7 100644
--- a/airflow/www/templates/airflow/list_dags.html
+++ b/airflow/www/templates/airflow/list_dags.html
@@ -172,7 +172,7 @@
<a href="{{ url_for("airflow.refresh", dag_id=row.dag_id) }}" title="Refresh">
<span class="glyphicon glyphicon-refresh" aria-hidden="true"></span>
</a>
- <a href="/admin/log/?sort=1&desc=1&flt1_dag_id_equals={{ row.dag_id }}" title="Logs">
+ <a href="{{ url_for('log.index_view') }}?sort=1&desc=1&flt1_dag_id_equals={{ row.dag_id }}" title="Logs">
<i class="icon-list"></i>
<span class="glyphicon glyphicon-align-justify" aria-hidden="true"></span>
</a>
@@ -276,7 +276,7 @@
})
.on('click', function(d, i) {
if (d.count > 0)
- window.location = "/admin/taskinstance/?flt1_dag_id_equals=" + d.dag_id + "&flt2_state_equals=" + d.state;
+ window.location = "{{ url_for('taskinstance.index_view') }}?flt1_dag_id_equals=" + d.dag_id + "&flt2_state_equals=" + d.state;
})
.on('mouseover', function(d, i) {
if (d.count > 0) {
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/www/templates/airflow/nvd3.html
----------------------------------------------------------------------
diff --git a/airflow/www/templates/airflow/nvd3.html b/airflow/www/templates/airflow/nvd3.html
index 5478ff8..45aca95 100644
--- a/airflow/www/templates/airflow/nvd3.html
+++ b/airflow/www/templates/airflow/nvd3.html
@@ -1,13 +1,13 @@
-{#
+{#
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -71,7 +71,7 @@ body {
<div id="container">
<h2>
<span id="label">{{ label }}</span>
- <a href="/admin/chart/edit/?id={{ chart.id }}" >
+ <a href="{{ url_for('chart.edit_view') }}?id={{ chart.id }}" >
<span class="glyphicon glyphicon-edit" aria-hidden="true" ></span>
</a>
</h2>
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/airflow/www/views.py
----------------------------------------------------------------------
diff --git a/airflow/www/views.py b/airflow/www/views.py
index cc73c8b..252241a 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -108,6 +108,9 @@ if conf.getboolean('webserver', 'FILTER_BY_OWNER'):
def dag_link(v, c, m, p):
+ if m.dag_id is None:
+ return Markup()
+
dag_id = bleach.clean(m.dag_id)
url = url_for(
'airflow.graph',
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/docs/integration.rst
----------------------------------------------------------------------
diff --git a/docs/integration.rst b/docs/integration.rst
index 142fca2..734ecad 100644
--- a/docs/integration.rst
+++ b/docs/integration.rst
@@ -1,12 +1,69 @@
Integration
===========
+- :ref:`ReverseProxy`
- :ref:`Azure`
- :ref:`AWS`
- :ref:`Databricks`
- :ref:`GCP`
+.. _ReverseProxy:
+Reverse Proxy
+-------------
+Airflow can be set up behind a reverse proxy, with the ability to set its endpoint with great
+flexibility.
+
+For example, you can configure your reverse proxy to get:
+
+::
+
+ https://lab.mycompany.com/myorg/airflow/
+
+To do so, you need to set the following setting in your `airflow.cfg`::
+
+ base_url = http://my_host/myorg/airflow
+
+Additionally if you use Celery Executor, you can get Flower in `/myorg/flower` with::
+
+ flower_url_prefix = /myorg/flower
+
+Your reverse proxy (ex: nginx) should be configured as follow:
+
+- pass the url and http header as it for the Airflow webserver, without any rewrite, for example::
+
+ server {
+ listen 80;
+ server_name lab.mycompany.com;
+
+ location /myorg/airflow/ {
+ proxy_pass http://localhost:8080;
+ proxy_set_header Host $host;
+ proxy_redirect off;
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
+ }
+ }
+
+- rewrite the url for the flower endpoint::
+
+ server {
+ listen 80;
+ server_name lab.mycompany.com;
+
+ location /myorg/flower/ {
+ rewrite ^/myorg/flower/(.*)$ /$1 break; # remove prefix from http header
+ proxy_pass http://localhost:5555;
+ proxy_set_header Host $host;
+ proxy_redirect off;
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
+ }
+ }
+
+
.. _Azure:
Azure: Microsoft Azure
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/setup.py
----------------------------------------------------------------------
diff --git a/setup.py b/setup.py
index 9358090..97fafe8 100644
--- a/setup.py
+++ b/setup.py
@@ -234,6 +234,7 @@ def do_setup():
'tabulate>=0.7.5, <0.8.0',
'thrift>=0.9.2',
'tzlocal>=1.4',
+ 'werkzeug>=0.14.1, <0.15.0',
'zope.deprecation>=4.0, <5.0',
],
setup_requires=[
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/1e36b37b/tests/www/test_views.py
----------------------------------------------------------------------
diff --git a/tests/www/test_views.py b/tests/www/test_views.py
index 6ea8db2..ff20333 100644
--- a/tests/www/test_views.py
+++ b/tests/www/test_views.py
@@ -21,6 +21,8 @@ import tempfile
import unittest
import sys
+from werkzeug.test import Client
+
from airflow import models, configuration, settings
from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
from airflow.models import DAG, TaskInstance
@@ -424,5 +426,25 @@ class TestVarImportView(unittest.TestCase):
self.assertIn('VALUE', body)
+class TestMountPoint(unittest.TestCase):
+ def setUp(self):
+ super(TestMountPoint, self).setUp()
+ configuration.load_test_config()
+ configuration.conf.set("webserver", "base_url", "http://localhost:8080/test")
+ config = dict()
+ config['WTF_CSRF_METHODS'] = []
+ app = application.cached_app(config=config, testing=True)
+ self.client = Client(app)
+
+ def test_mount(self):
+ response, _, _ = self.client.get('/', follow_redirects=True)
+ txt = b''.join(response)
+ self.assertEqual(b"Apache Airflow is not at this location", txt)
+
+ response, _, _ = self.client.get('/test', follow_redirects=True)
+ resp_html = b''.join(response)
+ self.assertIn(b"DAGs", resp_html)
+
+
if __name__ == '__main__':
unittest.main()