You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/06/26 17:23:58 UTC
[airflow] branch master updated: Add read-only Config endpoint
(#9497)
This is an automated email from the ASF dual-hosted git repository.
kamilbregula pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new f729cfd Add read-only Config endpoint (#9497)
f729cfd is described below
commit f729cfd3f29525dc958adfc8d0aa1f25c7f3044e
Author: zikun <33...@users.noreply.github.com>
AuthorDate: Sat Jun 27 01:23:20 2020 +0800
Add read-only Config endpoint (#9497)
Co-authored-by: Kamil Breguła <ka...@polidea.com>
Co-authored-by: Tomek Urbaszek <tu...@apache.org>
Co-authored-by: Kamil Breguła <mi...@users.noreply.github.com>
---
airflow/api_connexion/endpoints/config_endpoint.py | 63 ++++++++++++++++++--
airflow/api_connexion/openapi/v1.yaml | 3 -
airflow/api_connexion/schemas/config_schema.py | 57 ++++++++++++++++++
.../endpoints/test_config_endpoint.py | 67 +++++++++++++++++++---
tests/api_connexion/schemas/test_config_schema.py | 58 +++++++++++++++++++
5 files changed, 232 insertions(+), 16 deletions(-)
diff --git a/airflow/api_connexion/endpoints/config_endpoint.py b/airflow/api_connexion/endpoints/config_endpoint.py
index 1218d21..2261cac 100644
--- a/airflow/api_connexion/endpoints/config_endpoint.py
+++ b/airflow/api_connexion/endpoints/config_endpoint.py
@@ -15,12 +15,67 @@
# specific language governing permissions and limitations
# under the License.
-# TODO(mik-laj): We have to implement it.
-# Do you want to help? Please look at: https://github.com/apache/airflow/issues/8136
+from flask import Response, request
+from airflow.api_connexion.schemas.config_schema import Config, ConfigOption, ConfigSection, config_schema
+from airflow.configuration import conf
+from airflow.settings import json
-def get_config():
+LINE_SEP = '\n' # `\n` cannot appear in f-strings
+
+
+def _conf_dict_to_config(conf_dict: dict) -> Config:
+ """Convert config dict to a Config object"""
+ config = Config(
+ sections=[
+ ConfigSection(
+ name=section,
+ options=[
+ ConfigOption(key=key, value=value)
+ for key, value in options.items()
+ ]
+ )
+ for section, options in conf_dict.items()
+ ]
+ )
+ return config
+
+
+def _option_to_text(config_option: ConfigOption) -> str:
+ """Convert a single config option to text"""
+ return f'{config_option.key} = {config_option.value}'
+
+
+def _section_to_text(config_section: ConfigSection) -> str:
+ """Convert a single config section to text"""
+ return (f'[{config_section.name}]{LINE_SEP}'
+ f'{LINE_SEP.join(_option_to_text(option) for option in config_section.options)}{LINE_SEP}')
+
+
+def _config_to_text(config: Config) -> str:
+ """Convert the entire config to text"""
+ return LINE_SEP.join(_section_to_text(s) for s in config.sections)
+
+
+def _config_to_json(config: Config) -> str:
+ """Convert a Config object to a JSON formatted string"""
+ return json.dumps(config_schema.dump(config).data, indent=4)
+
+
+def get_config() -> Response:
"""
Get current configuration.
"""
- raise NotImplementedError("Not implemented yet.")
+ serializer = {
+ 'text/plain': _config_to_text,
+ 'application/json': _config_to_json,
+ }
+ response_types = serializer.keys()
+ return_type = request.accept_mimetypes.best_match(response_types)
+ conf_dict = conf.as_dict(display_source=False, display_sensitive=True)
+ config = _conf_dict_to_config(conf_dict)
+ if return_type not in serializer:
+ return Response(status=406)
+ else:
+ config_text = serializer[return_type](config)
+ return Response(config_text, headers={'Content-Type': return_type})
diff --git a/airflow/api_connexion/openapi/v1.yaml b/airflow/api_connexion/openapi/v1.yaml
index 1794b65..eefcc0f 100644
--- a/airflow/api_connexion/openapi/v1.yaml
+++ b/airflow/api_connexion/openapi/v1.yaml
@@ -1133,9 +1133,6 @@ paths:
x-openapi-router-controller: airflow.api_connexion.endpoints.config_endpoint
operationId: get_config
tags: [Config]
- parameters:
- - $ref: '#/components/parameters/PageLimit'
- - $ref: '#/components/parameters/PageOffset'
responses:
'200':
description: Return current configuration.
diff --git a/airflow/api_connexion/schemas/config_schema.py b/airflow/api_connexion/schemas/config_schema.py
new file mode 100644
index 0000000..4af3db6
--- /dev/null
+++ b/airflow/api_connexion/schemas/config_schema.py
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import List, NamedTuple
+
+from marshmallow import Schema, fields
+
+
+class ConfigOptionSchema(Schema):
+ """ Config Option Schema """
+ key = fields.String(required=True)
+ value = fields.String(required=True)
+
+
+class ConfigOption(NamedTuple):
+ """ Config option """
+ key: str
+ value: str
+
+
+class ConfigSectionSchema(Schema):
+ """ Config Section Schema"""
+ name = fields.String(required=True)
+ options = fields.List(fields.Nested(ConfigOptionSchema))
+
+
+class ConfigSection(NamedTuple):
+ """ List of config options within a section """
+ name: str
+ options: List[ConfigOption]
+
+
+class ConfigSchema(Schema):
+ """ Config Schema"""
+ sections = fields.List(fields.Nested(ConfigSectionSchema))
+
+
+class Config(NamedTuple):
+ """ List of config sections with their options """
+ sections: List[ConfigSection]
+
+
+config_schema = ConfigSchema(strict=True)
diff --git a/tests/api_connexion/endpoints/test_config_endpoint.py b/tests/api_connexion/endpoints/test_config_endpoint.py
index d0c9efb..13d59e8 100644
--- a/tests/api_connexion/endpoints/test_config_endpoint.py
+++ b/tests/api_connexion/endpoints/test_config_endpoint.py
@@ -14,23 +14,72 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import unittest
-import pytest
+import textwrap
+
+from mock import patch
from airflow.www import app
+MOCK_CONF = {
+ 'core': {
+ 'parallelism': '1024',
+ },
+ 'smtp': {
+ 'smtp_host': 'localhost',
+ 'smtp_mail_from': 'airflow@example.com',
+ },
+}
+
-class TestGetConfig(unittest.TestCase):
+@patch(
+ "airflow.api_connexion.endpoints.config_endpoint.conf.as_dict",
+ return_value=MOCK_CONF
+)
+class TestGetConfig:
@classmethod
- def setUpClass(cls) -> None:
- super().setUpClass()
+ def setup_class(cls) -> None:
cls.app = app.create_app(testing=True) # type:ignore
+ cls.client = None
- def setUp(self) -> None:
+ def setup_method(self) -> None:
self.client = self.app.test_client() # type:ignore
- @pytest.mark.skip(reason="Not implemented yet")
- def test_should_response_200(self):
- response = self.client.get("/api/v1/config")
+ def test_should_response_200_text_plain(self, mock_as_dict):
+ response = self.client.get("/api/v1/config", headers={'Accept': 'text/plain'})
+ assert response.status_code == 200
+ expected = textwrap.dedent("""\
+ [core]
+ parallelism = 1024
+
+ [smtp]
+ smtp_host = localhost
+ smtp_mail_from = airflow@example.com
+ """)
+ assert expected == response.data.decode()
+
+ def test_should_response_200_application_json(self, mock_as_dict):
+ response = self.client.get("/api/v1/config", headers={'Accept': 'application/json'})
assert response.status_code == 200
+ expected = {
+ 'sections': [
+ {
+ 'name': 'core',
+ 'options': [
+ {'key': 'parallelism', 'value': '1024'},
+ ]
+ },
+ {
+ 'name': 'smtp',
+ 'options': [
+ {'key': 'smtp_host', 'value': 'localhost'},
+ {'key': 'smtp_mail_from', 'value': 'airflow@example.com'},
+ ]
+ },
+ ]
+ }
+ assert expected == response.json
+
+ def test_should_response_406(self, mock_as_dict):
+ response = self.client.get("/api/v1/config", headers={'Accept': 'application/octet-stream'})
+ assert response.status_code == 406
diff --git a/tests/api_connexion/schemas/test_config_schema.py b/tests/api_connexion/schemas/test_config_schema.py
new file mode 100644
index 0000000..3494159
--- /dev/null
+++ b/tests/api_connexion/schemas/test_config_schema.py
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.api_connexion.schemas.config_schema import Config, ConfigOption, ConfigSection, config_schema
+
+
+class TestConfigSchema:
+ def test_serialize(self):
+ config = Config(
+ sections=[
+ ConfigSection(
+ name='sec1',
+ options=[
+ ConfigOption(key='apache', value='airflow'),
+ ConfigOption(key='hello', value='world'),
+ ]
+ ),
+ ConfigSection(
+ name='sec2',
+ options=[
+ ConfigOption(key='foo', value='bar'),
+ ]
+ ),
+ ]
+ )
+ result = config_schema.dump(config)
+ expected = {
+ 'sections': [
+ {
+ 'name': 'sec1',
+ 'options': [
+ {'key': 'apache', 'value': 'airflow'},
+ {'key': 'hello', 'value': 'world'},
+ ]
+ },
+ {
+ 'name': 'sec2',
+ 'options': [
+ {'key': 'foo', 'value': 'bar'},
+ ]
+ },
+ ]
+ }
+ assert result.data == expected