You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/11/23 03:31:27 UTC
[airflow] branch master updated: Add capability to specify gunicorn
access log format (#10261)
This is an automated email from the ASF dual-hosted git repository.
kamilbregula pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new ff990f2 Add capability to specify gunicorn access log format (#10261)
ff990f2 is described below
commit ff990f245eefd24f1a9a16d2b1712103554f870f
Author: Aman Ranjan Thakur <am...@gmail.com>
AuthorDate: Mon Nov 23 08:59:10 2020 +0530
Add capability to specify gunicorn access log format (#10261)
---
airflow/cli/cli_parser.py | 6 ++++
airflow/cli/commands/webserver_command.py | 6 ++++
airflow/config_templates/config.yml | 9 +++++
airflow/config_templates/default_airflow.cfg | 5 +++
docs/spelling_wordlist.txt | 1 +
tests/cli/commands/test_webserver_command.py | 52 ++++++++++++++++++++++++++++
6 files changed, 79 insertions(+)
diff --git a/airflow/cli/cli_parser.py b/airflow/cli/cli_parser.py
index 46addc1..a2d31c8 100644
--- a/airflow/cli/cli_parser.py
+++ b/airflow/cli/cli_parser.py
@@ -458,6 +458,11 @@ ARG_ERROR_LOGFILE = Arg(
default=conf.get('webserver', 'ERROR_LOGFILE'),
help="The logfile to store the webserver error log. Use '-' to print to stderr",
)
+ARG_ACCESS_LOGFORMAT = Arg(
+ ("-L", "--access-logformat"),
+ default=conf.get('webserver', 'ACCESS_LOGFORMAT'),
+ help="The access log format for gunicorn logs",
+)
# scheduler
ARG_NUM_RUNS = Arg(
@@ -1343,6 +1348,7 @@ airflow_commands: List[CLICommand] = [
ARG_STDERR,
ARG_ACCESS_LOGFILE,
ARG_ERROR_LOGFILE,
+ ARG_ACCESS_LOGFORMAT,
ARG_LOG_FILE,
ARG_SSL_CERT,
ARG_SSL_KEY,
diff --git a/airflow/cli/commands/webserver_command.py b/airflow/cli/commands/webserver_command.py
index 9e82620..7be3c33 100644
--- a/airflow/cli/commands/webserver_command.py
+++ b/airflow/cli/commands/webserver_command.py
@@ -319,6 +319,7 @@ def webserver(args):
access_logfile = args.access_logfile or conf.get('webserver', 'access_logfile')
error_logfile = args.error_logfile or conf.get('webserver', 'error_logfile')
+ access_logformat = args.access_logformat or conf.get('webserver', 'access_logformat')
num_workers = args.workers or conf.get('webserver', 'workers')
worker_timeout = args.worker_timeout or conf.get('webserver', 'web_server_worker_timeout')
ssl_cert = args.ssl_cert or conf.get('webserver', 'web_server_ssl_cert')
@@ -360,6 +361,7 @@ def webserver(args):
Host: {hostname}:{port}
Timeout: {worker_timeout}
Logfiles: {access_logfile} {error_logfile}
+ Access Logformat: {access_logformat}
=================================================================\
'''.format(
num_workers=num_workers,
@@ -369,6 +371,7 @@ def webserver(args):
worker_timeout=worker_timeout,
access_logfile=access_logfile,
error_logfile=error_logfile,
+ access_logformat=access_logformat,
)
)
)
@@ -397,6 +400,9 @@ def webserver(args):
if args.error_logfile:
run_args += ['--error-logfile', str(args.error_logfile)]
+ if args.access_logformat and args.access_logformat.strip():
+ run_args += ['--access-logformat', str(args.access_logformat)]
+
if args.daemon:
run_args += ['--daemon']
diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml
index 516bb62..92fee3a4 100644
--- a/airflow/config_templates/config.yml
+++ b/airflow/config_templates/config.yml
@@ -976,6 +976,15 @@
type: string
example: ~
default: "-"
+ - name: access_logformat
+ description: |
+ Access log format for gunicorn webserver.
+ default format is %%(h)s %%(l)s %%(u)s %%(t)s "%%(r)s" %%(s)s %%(b)s "%%(f)s" "%%(a)s"
+ documentation - https://docs.gunicorn.org/en/stable/settings.html#access-log-format
+ version_added: ~
+ type: string
+ example: ~
+ default: ""
- name: expose_config
description: |
Expose the configuration file in the web server
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index cebbfd9..c475156 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -492,6 +492,11 @@ access_logfile = -
# Log files for the gunicorn webserver. '-' means log to stderr.
error_logfile = -
+# Access log format for gunicorn webserver.
+# default format is %%(h)s %%(l)s %%(u)s %%(t)s "%%(r)s" %%(s)s %%(b)s "%%(f)s" "%%(a)s"
+# documentation - https://docs.gunicorn.org/en/stable/settings.html#access-log-format
+access_logformat =
+
# Expose the configuration file in the web server
expose_config = False
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index c4fadc8..40e3b4e 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -916,6 +916,7 @@ localhost
localstack
lodash
logfile
+logformat
loglevel
logstash
lshift
diff --git a/tests/cli/commands/test_webserver_command.py b/tests/cli/commands/test_webserver_command.py
index 81fe5a3..f5bc517 100644
--- a/tests/cli/commands/test_webserver_command.py
+++ b/tests/cli/commands/test_webserver_command.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import json
import os
import subprocess
import tempfile
@@ -378,3 +379,54 @@ class TestCliWebServer(unittest.TestCase):
)
proc.terminate()
self.assertEqual(-15, proc.wait(60))
+
+ def test_cli_webserver_access_log_format(self):
+
+ # json access log format
+ access_logformat = (
+ "{\"ts\":\"%(t)s\",\"remote_ip\":\"%(h)s\",\"request_id\":\"%({"
+ "X-Request-Id}i)s\",\"code\":\"%(s)s\",\"request_method\":\"%(m)s\","
+ "\"request_path\":\"%(U)s\",\"agent\":\"%(a)s\",\"response_time\":\"%(D)s\","
+ "\"response_length\":\"%(B)s\"} "
+ )
+ with tempfile.TemporaryDirectory() as tmpdir, mock.patch.dict(
+ "os.environ",
+ AIRFLOW__CORE__DAGS_FOLDER="/dev/null",
+ AIRFLOW__CORE__LOAD_EXAMPLES="False",
+ AIRFLOW__WEBSERVER__WORKERS="1",
+ ):
+ access_logfile = f"{tmpdir}/access.log"
+ # Run webserver in foreground and terminate it.
+ proc = subprocess.Popen(
+ [
+ "airflow",
+ "webserver",
+ "--access-logfile",
+ access_logfile,
+ "--access-logformat",
+ access_logformat,
+ ]
+ )
+ self.assertEqual(None, proc.poll())
+
+ # Wait for webserver process
+ sleep(10)
+
+ proc2 = subprocess.Popen(["curl", "http://localhost:8080"])
+ proc2.wait(10)
+ try:
+ file = open(access_logfile)
+ log = json.loads(file.read())
+ self.assertEqual('127.0.0.1', log.get('remote_ip'))
+ self.assertEqual(len(log), 9)
+ self.assertEqual('GET', log.get('request_method'))
+
+ except OSError:
+ print("access log file not found at " + access_logfile)
+
+ # Terminate webserver
+ proc.terminate()
+ # -15 - the server was stopped before it started
+ # 0 - the server terminated correctly
+ self.assertIn(proc.wait(60), (-15, 0))
+ self._check_processes()