You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by "ASF GitHub Bot (JIRA)" <ji...@apache.org> on 2018/11/01 21:12:00 UTC
[jira] [Commented] (AIRFLOW-1371) Customize log_filepath to allow
files writen to NFS-shares
[ https://issues.apache.org/jira/browse/AIRFLOW-1371?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16672209#comment-16672209 ]
ASF GitHub Bot commented on AIRFLOW-1371:
-----------------------------------------
ashb closed pull request #2449: [AIRFLOW-1371] Customize log_filepath to allow files writen to NFS-shares
URL: https://github.com/apache/incubator-airflow/pull/2449
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py
index 4b3a0edc8f..d5b83d6268 100755
--- a/airflow/bin/cli.py
+++ b/airflow/bin/cli.py
@@ -361,8 +361,9 @@ def run(args, dag=None):
# Create the directory as globally writable using custom mkdirs
# as os.makedirs doesn't set mode properly.
mkdirs(directory, 0o775)
- iso = args.execution_date.isoformat()
- filename = "{directory}/{iso}".format(**locals())
+ pattern = conf.get('core', 'LOG_FILE_DATETIME_PATTERN')
+ log_file = args.execution_date.strftime(pattern)
+ filename = "{directory}/{log_file}".format(**locals())
if not os.path.exists(filename):
open(filename, "a").close()
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index ddd1ba816e..85fd83cf50 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -55,6 +55,11 @@ log_format_with_pid = [%%(asctime)s] [%%(process)d] {{%%(filename)s:%%(lineno)d}
log_format_with_thread_name = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(threadName)s %%(levelname)s - %%(message)s
simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s
+# Log filename format
+# used by airflow to create task-runs log files based on the execution_date,
+# some filesystems like NFS have colon as a restricted char
+log_file_datetime_pattern = %%Y-%%m-%%dT%%H:%%M:%%S
+
# The executor class that airflow should use. Choices include
# SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor
executor = SequentialExecutor
diff --git a/airflow/models.py b/airflow/models.py
index 32ad144a22..ae1c3b403c 100755
--- a/airflow/models.py
+++ b/airflow/models.py
@@ -939,10 +939,15 @@ def generate_command(dag_id,
@property
def log_filepath(self):
- iso = self.execution_date.isoformat()
- log = os.path.expanduser(configuration.get('core', 'BASE_LOG_FOLDER'))
- return (
- "{log}/{self.dag_id}/{self.task_id}/{iso}.log".format(**locals()))
+ base_log_path = os.path.expanduser(
+ configuration.get('core', 'BASE_LOG_FOLDER')
+ )
+ pattern = configuration.get('core', 'LOG_FILE_DATETIME_PATTERN')
+ log_file = self.execution_date.strftime(pattern)
+ return ("{base_log_path}/"
+ "{self.dag_id}/"
+ "{self.task_id}/"
+ "{log_file}.log").format(**locals())
@property
def log_url(self):
diff --git a/airflow/www/views.py b/airflow/www/views.py
index 6c39462072..fb2f56274a 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -700,17 +700,20 @@ def rendered(self):
def log(self):
BASE_LOG_FOLDER = os.path.expanduser(
conf.get('core', 'BASE_LOG_FOLDER'))
+ pattern = conf.get('core', 'LOG_FILE_DATETIME_PATTERN')
+
dag_id = request.args.get('dag_id')
task_id = request.args.get('task_id')
execution_date = request.args.get('execution_date')
+
+ dttm = dateutil.parser.parse(execution_date)
+ log_filename = dttm.strftime(pattern)
+ log_relative = "{dag_id}/{task_id}/{log_filename}".format(**locals())
dag = dagbag.get_dag(dag_id)
- log_relative = "{dag_id}/{task_id}/{execution_date}".format(
- **locals())
loc = os.path.join(BASE_LOG_FOLDER, log_relative)
loc = loc.format(**locals())
log = ""
TI = models.TaskInstance
- dttm = dateutil.parser.parse(execution_date)
form = DateTimeForm(data={'execution_date': dttm})
session = Session()
ti = session.query(TI).filter(
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
> Customize log_filepath to allow files writen to NFS-shares
> ----------------------------------------------------------
>
> Key: AIRFLOW-1371
> URL: https://issues.apache.org/jira/browse/AIRFLOW-1371
> Project: Apache Airflow
> Issue Type: Improvement
> Components: models
> Affects Versions: 1.8.0
> Reporter: Alexander Bij
> Priority: Minor
>
> To be able to send logs to a NFS-mount the folder and files cannot contain [reserved characters | https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx] like:
> * < (less than)
> * > (greater than)
> * : (colon)
> * " (double quote)
> * / (forward slash)
> * \ (backslash)
> * ? (question mark)
> * (asterisk)
> Airflow writes the logs with an ISO-datetime format. This file contains one of these characters.
> There are some solutions. What I would like is a configurable log_file_format to create log-files.
> {code}
> # Fetched by views.py and written by models.py currently like this:
> @property
> def log_filepath(self):
> iso = self.execution_date.isoformat()
> log = os.path.expanduser(configuration.get('core', 'BASE_LOG_FOLDER'))
> return "{log}/{self.dag_id}/{self.task_id}/{iso}.log".format(**locals()))
> {code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)