You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@httpd.apache.org by Thomas Meyer <th...@m3y3r.de> on 2023/03/24 21:54:08 UTC
[RFC] JSON logging support for httpd 2.4.x
Hi,
please have a look at this preliminarily work to support JSON output in mod_log_config.
It's still unfinished and has probably a lot of bugs, but this is to show the general idea
of my solution.
Help and feedback is most welcome.
Mfg
thomas
[PATCH 3/3] mod_log_config: Add JSON logger
Posted by Thomas Meyer <th...@m3y3r.de>.
---
modules/loggers/mod_log_config.c | 159 +++++++++++++++++++++++++++++--
1 file changed, 153 insertions(+), 6 deletions(-)
diff --git a/modules/loggers/mod_log_config.c b/modules/loggers/mod_log_config.c
index d142c888ad..188131ebac 100644
--- a/modules/loggers/mod_log_config.c
+++ b/modules/loggers/mod_log_config.c
@@ -179,7 +179,10 @@ module AP_MODULE_DECLARE_DATA log_config_module;
static int xfer_flags = (APR_WRITE | APR_APPEND | APR_CREATE | APR_LARGEFILE);
static apr_fileperms_t xfer_perms = APR_OS_DEFAULT;
-static apr_hash_t *log_hash;
+
+static apr_hash_t *log_hash; // tag to log_struct
+static apr_hash_t *json_hash; // tag to json attribute name
+
static apr_status_t ap_default_log_writer(request_rec *r,
void *handle,
const char **strs,
@@ -194,6 +197,14 @@ static apr_status_t ap_buffered_log_writer(request_rec *r,
int nelts,
void *items,
apr_size_t len);
+static apr_status_t ap_json_log_writer(request_rec *r,
+ void *handle,
+ const char **strs,
+ int *strl,
+ int nelts,
+ void *items,
+ apr_size_t len);
+
static void *ap_default_log_writer_init(apr_pool_t *p, server_rec *s,
const char* name);
static void *ap_buffered_log_writer_init(apr_pool_t *p, server_rec *s,
@@ -287,11 +298,11 @@ typedef struct {
*/
typedef struct {
- char *tag; /* tag that did create this lfi */
ap_log_handler_fn_t *func;
char *arg;
int condition_sense;
int want_orig;
+ char *tag; /* tag that did create this lfi */
apr_array_header_t *conditions;
} log_format_item;
@@ -967,6 +978,7 @@ static char *parse_log_item(apr_pool_t *p, log_format_item *it, const char **sa)
it->want_orig = -1;
it->arg = ""; /* For safety's sake... */
+ it->tag = NULL;
while (*s) {
int i;
@@ -1026,16 +1038,16 @@ static char *parse_log_item(apr_pool_t *p, log_format_item *it, const char **sa)
}
}
if (!handler) {
- handler = (ap_log_handler *)apr_hash_get(log_hash, s++, 1);
+ handler = (ap_log_handler *)apr_hash_get(log_hash, s, 1);
if (!handler) {
char dummy[2];
- dummy[0] = s[-1];
+ dummy[0] = s[0];
dummy[1] = '\0';
return apr_pstrcat(p, "Unrecognized LogFormat directive %",
dummy, NULL);
}
- it->tag=apr_pstrmemdup(p, s, 1);
+ it->tag=apr_pstrmemdup(p, s++, 1);
}
it->func = handler->func;
if (it->want_orig == -1) {
@@ -1378,6 +1390,17 @@ static const char *set_transfer_log(cmd_parms *cmd, void *dummy,
return add_custom_log(cmd, dummy, fn, NULL, NULL);
}
+static const char *set_json_logs_on(cmd_parms *parms, void *dummy, int flag)
+{
+ if (flag) {
+ ap_log_set_writer(ap_json_log_writer);
+ }
+ else {
+ ap_log_set_writer(ap_default_log_writer);
+ }
+ return NULL;
+}
+
static const char *set_buffered_logs_on(cmd_parms *parms, void *dummy, int flag)
{
buffered_logs = flag;
@@ -1391,6 +1414,7 @@ static const char *set_buffered_logs_on(cmd_parms *parms, void *dummy, int flag)
}
return NULL;
}
+
static const command_rec config_log_cmds[] =
{
AP_INIT_TAKE23("CustomLog", add_custom_log, NULL, RSRC_CONF,
@@ -1404,6 +1428,8 @@ AP_INIT_TAKE12("LogFormat", log_format, NULL, RSRC_CONF,
"a log format string (see docs) and an optional format name"),
AP_INIT_FLAG("BufferedLogs", set_buffered_logs_on, NULL, RSRC_CONF,
"Enable Buffered Logging (experimental)"),
+AP_INIT_FLAG("JsonLogs", set_json_logs_on, NULL, RSRC_CONF,
+ "Enable JSON Logging (experimental)"),
{NULL}
};
@@ -1608,6 +1634,84 @@ static ap_log_writer *ap_log_set_writer(ap_log_writer *handle)
return old;
}
+/* see https://www.rfc-editor.org/rfc/rfc8259#section-7 */
+static int json_needs_encoding(const char* string)
+{
+ for(int i = 0, n = strlen(string); i < n; i++) {
+ char c = string[i];
+ if(c < 0x20 || c == 0x22 || c == 0x5c) {
+ return 1; // true
+ }
+ }
+
+ return 0; // false
+}
+
+static const char* json_encode(apr_pool_t *p, const char* utf8_string_to_encode)
+{
+ for(int i = 0, n = strlen(utf8_string_to_encode); i < n; i++) {
+ char c = utf8_string_to_encode[i];
+ if(c < 0x20 || c == 0x22 || c == 0x5c) {
+ return 1; // true
+ }
+ }
+
+ return 0; // false
+}
+
+static apr_status_t ap_json_log_writer( request_rec *r,
+ void *handle,
+ const char **strs,
+ int *strl,
+ int nelts,
+ void *itms,
+ apr_size_t len)
+
+{
+ log_format_item *items = (log_format_item *) itms;
+ apr_size_t len_file_write;
+ const char* attribute_name;
+ const char* attribute_value;
+ apr_size_t json_str_total_len = len * 4;
+ char *json_str = apr_palloc(r->pool, json_str_total_len + 1); //TODO: Why can't this fail?
+
+ // build json
+ apr_cpystrn(json_str, "{", json_str_total_len);
+ for (int i = 0; i < nelts; ++i) {
+ if(items[i].tag == NULL) {
+ continue;
+ }
+
+ attribute_name = apr_hash_get(json_hash, items[i].tag, APR_HASH_KEY_STRING );
+ if(!attribute_name) {
+ attribute_name = items[i].tag; // use tag as attribute name as fallback
+
+ // TODO: do we really needs to check for json string encoding for tags?
+ if(json_needs_encoding(attribute_name)) {
+ attribute_name = json_encode(r->pool, attribute_name);
+ }
+ }
+ // TODO: enhance attribute_name with argument from log_format_item in case of {...}
+
+ strncat(json_str, "\"", json_str_total_len - strnlen(json_str, json_str_total_len));
+ strncat(json_str, attribute_name, json_str_total_len - strnlen(json_str, json_str_total_len));
+ strncat(json_str, "\":\"", json_str_total_len - strnlen(json_str, json_str_total_len));
+
+ attribute_value = strs[i];
+ if(json_needs_encoding(attribute_value)) {
+ attribute_value = json_encode(r->pool, attribute_value);
+ }
+ strncat(json_str, attribute_value, json_str_total_len - strnlen(json_str, json_str_total_len));
+ strncat(json_str, "\",", json_str_total_len - strnlen(json_str, json_str_total_len));
+ }
+ // remove last ',' again
+ json_str[strnlen(json_str, json_str_total_len) - 1] = '\0';
+ strncat(json_str, "}" APR_EOL_STR, json_str_total_len - strnlen(json_str, json_str_total_len));
+
+ len_file_write = strnlen(json_str, json_str_total_len);
+ return apr_file_write((apr_file_t*)handle, json_str, &len_file_write);
+}
+
static apr_status_t ap_default_log_writer( request_rec *r,
void *handle,
const char **strs,
@@ -1615,7 +1719,6 @@ static apr_status_t ap_default_log_writer( request_rec *r,
int nelts,
void *items,
apr_size_t len)
-
{
char *str;
char *s;
@@ -1733,6 +1836,15 @@ static apr_status_t ap_buffered_log_writer(request_rec *r,
return rv;
}
+static void json_register_attribute(apr_pool_t *p, const char *tag, const char* attribute_name)
+{
+ if(json_needs_encoding(attribute_name)) {
+ attribute_name = json_encode(p, attribute_name);
+ }
+
+ apr_hash_set(json_hash, tag, strlen(tag), (const void *)attribute_name);
+}
+
static int log_pre_config(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp)
{
static APR_OPTIONAL_FN_TYPE(ap_register_log_handler) *log_pfn_register;
@@ -1775,6 +1887,40 @@ static int log_pre_config(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp)
log_pfn_register(p, "^to", log_trailer_out, 0);
}
+ // TODO: align attribute names with https://github.com/apache/tomcat/commit/00edb6d271f6ffbe65a01acc377b1930c7354ab0
+ if(1) {
+ json_register_attribute(p, "h", "host");
+ json_register_attribute(p, "a", "remoteAddr");
+ json_register_attribute(p, "A", "localAddr");
+ json_register_attribute(p, "l", "logicalUserName");
+ json_register_attribute(p, "u", "user");
+ json_register_attribute(p, "t", "time");
+ json_register_attribute(p, "f", "file");
+ json_register_attribute(p, "b", "size");
+ json_register_attribute(p, "B", "byteSentNC");
+ json_register_attribute(p, "i", "headerIn");
+ json_register_attribute(p, "o", "headerOut");
+ json_register_attribute(p, "n", "note");
+ json_register_attribute(p, "L", "logId");
+ json_register_attribute(p, "e", "env");
+ json_register_attribute(p, "V", "serverName");
+ json_register_attribute(p, "v", "virtualHost");
+ json_register_attribute(p, "p", "port");
+ json_register_attribute(p, "P", "threadId");
+ json_register_attribute(p, "H", "protocol");
+ json_register_attribute(p, "m", "method");
+ json_register_attribute(p, "q", "query");
+ json_register_attribute(p, "X", "connectionStatus");
+ json_register_attribute(p, "C", "cookie");
+ json_register_attribute(p, "k", "requestsOnConnection");
+ json_register_attribute(p, "r", "request");
+ json_register_attribute(p, "D", "elapsedTime");
+ json_register_attribute(p, "T", "elapsedTimeS");
+ json_register_attribute(p, "U", "path");
+ json_register_attribute(p, "s", "statusCode");
+ json_register_attribute(p, "R", "handler");
+ }
+
/* reset to default conditions */
ap_log_set_writer_init(ap_default_log_writer_init);
ap_log_set_writer(ap_default_log_writer);
@@ -1847,6 +1993,7 @@ static void register_hooks(apr_pool_t *p)
* before calling APR_REGISTER_OPTIONAL_FN.
*/
log_hash = apr_hash_make(p);
+ json_hash = apr_hash_make(p);
APR_REGISTER_OPTIONAL_FN(ap_register_log_handler);
APR_REGISTER_OPTIONAL_FN(ap_log_set_writer_init);
APR_REGISTER_OPTIONAL_FN(ap_log_set_writer);
--
2.20.1
[PATCH 1/3] mod_log_config: add creating tag to log_format_item
Posted by Thomas Meyer <th...@m3y3r.de>.
Preprartional change for upcoming JSON support
---
modules/loggers/mod_log_config.c | 26 +++++++++++++++-----------
1 file changed, 15 insertions(+), 11 deletions(-)
diff --git a/modules/loggers/mod_log_config.c b/modules/loggers/mod_log_config.c
index 5d5b73a1f5..6934a8691c 100644
--- a/modules/loggers/mod_log_config.c
+++ b/modules/loggers/mod_log_config.c
@@ -285,6 +285,7 @@ typedef struct {
*/
typedef struct {
+ char *tag; /* tag that did create this lfi */
ap_log_handler_fn_t *func;
char *arg;
int condition_sense;
@@ -884,6 +885,7 @@ static char *parse_log_misc_string(apr_pool_t *p, log_format_item *it,
const char *s;
char *d;
+ it->tag = NULL;
it->func = constant_item;
it->conditions = NULL;
@@ -1014,22 +1016,24 @@ static char *parse_log_item(apr_pool_t *p, log_format_item *it, const char **sa)
default:
/* check for '^' + two character format first */
- if (*s == '^' && *(s+1) && *(s+2)) {
+ if (*s == '^' && *(s+1) && *(s+2)) {
handler = (ap_log_handler *)apr_hash_get(log_hash, s, 3);
- if (handler) {
+ if (handler) {
+ it->tag=apr_pstrmemdup(p, s, 3);
s += 3;
}
}
if (!handler) {
- handler = (ap_log_handler *)apr_hash_get(log_hash, s++, 1);
- }
- if (!handler) {
- char dummy[2];
-
- dummy[0] = s[-1];
- dummy[1] = '\0';
- return apr_pstrcat(p, "Unrecognized LogFormat directive %",
- dummy, NULL);
+ handler = (ap_log_handler *)apr_hash_get(log_hash, s++, 1);
+ if (!handler) {
+ char dummy[2];
+
+ dummy[0] = s[-1];
+ dummy[1] = '\0';
+ return apr_pstrcat(p, "Unrecognized LogFormat directive %",
+ dummy, NULL);
+ }
+ it->tag=apr_pstrmemdup(p, s, 1);
}
it->func = handler->func;
if (it->want_orig == -1) {
--
2.20.1
Re: [RFC] JSON logging support for httpd 2.4.x
Posted by Thomas Meyer <th...@m3y3r.de>.
Hi,
How to proceed with #373, can someone please give some feedback?
Mfg
Thomas
Am 8. September 2023 20:35:09 MESZ schrieb Thomas Meyer <th...@m3y3r.de>:
>Hi,
>
>as I learned all code must go to trunk first, so I kindly ask to review my updated patch against trunk here:
>
>https://github.com/apache/httpd/pull/373
>
>Once this pr is reviewed and merged I'm going to retrofit pr353 against 2.4.x
>
>with kind regards
>thomas
>
>Am 31. März 2023 09:04:35 MESZ schrieb Thomas Meyer <th...@m3y3r.de>:
>>Hi,
>>
>>Sadly I got no feedback at all.
>>
>>What is the preferred way for contributions?
>>
>>I also did raise a PR here with some fixes on top of this patch series:
>>
>>https://github.com/apache/httpd/pull/353
>>
>>Mfg
>>Thomas
>>
>>Am 24. März 2023 22:54:08 MEZ schrieb Thomas Meyer <th...@m3y3r.de>:
>>>
>>>Hi,
>>>
>>>please have a look at this preliminarily work to support JSON output in mod_log_config.
>>>
>>>It's still unfinished and has probably a lot of bugs, but this is to show the general idea
>>>of my solution.
>>>
>>>Help and feedback is most welcome.
>>>
>>>Mfg
>>>thomas
>>>
>>>
>>
>>--
>>Diese Nachricht wurde von meinem Android-Gerät mit K-9 Mail gesendet.
>--
>Diese Nachricht wurde von meinem Android-Gerät mit K-9 Mail gesendet.
--
Diese Nachricht wurde von meinem Android-Gerät mit K-9 Mail gesendet.
Re: [RFC] JSON logging support for httpd 2.4.x
Posted by Thomas Meyer <th...@m3y3r.de>.
Hi,
as I learned all code must go to trunk first, so I kindly ask to review my updated patch against trunk here:
https://github.com/apache/httpd/pull/373
Once this pr is reviewed and merged I'm going to retrofit pr353 against 2.4.x
with kind regards
thomas
Am 31. März 2023 09:04:35 MESZ schrieb Thomas Meyer <th...@m3y3r.de>:
>Hi,
>
>Sadly I got no feedback at all.
>
>What is the preferred way for contributions?
>
>I also did raise a PR here with some fixes on top of this patch series:
>
>https://github.com/apache/httpd/pull/353
>
>Mfg
>Thomas
>
>Am 24. März 2023 22:54:08 MEZ schrieb Thomas Meyer <th...@m3y3r.de>:
>>
>>Hi,
>>
>>please have a look at this preliminarily work to support JSON output in mod_log_config.
>>
>>It's still unfinished and has probably a lot of bugs, but this is to show the general idea
>>of my solution.
>>
>>Help and feedback is most welcome.
>>
>>Mfg
>>thomas
>>
>>
>
>--
>Diese Nachricht wurde von meinem Android-Gerät mit K-9 Mail gesendet.
--
Diese Nachricht wurde von meinem Android-Gerät mit K-9 Mail gesendet.
Re: [RFC] JSON logging support for httpd 2.4.x
Posted by Christophe JAILLET <ch...@wanadoo.fr>.
Le 31/03/2023 à 09:04, Thomas Meyer a écrit :
> Hi,
>
> Sadly I got no feedback at all.
>
> What is the preferred way for contributions?
Hi,
mailing list is fine.
You can also use github if it is easier for you.
>
> I also did raise a PR here with some fixes on top of this patch series:
>
> https://github.com/apache/httpd/pull/353
> <https://github.com/apache/httpd/pull/353>
>
> Mfg
> Thomas
>
> Am 24. März 2023 22:54:08 MEZ schrieb Thomas Meyer <th...@m3y3r.de>:
>
>
> Hi,
>
> please have a look at this preliminarily work to support JSON output in mod_log_config.
>
> It's still unfinished and has probably a lot of bugs, but this is to show the general idea
> of my solution.
>
> Help and feedback is most welcome.
>
> Mfg
> thomas
>
>
> --
> Diese Nachricht wurde von meinem Android-Gerät mit K-9 Mail gesendet.
As you have already notices, there is already a mod_log_json in trunk
(see [1])
AFAIK, it is not documented.
Have you looked at it first?
Does it match your needs?
There is also an old WiP related to make ErrorLog logging modular.
I've not looked at it since a LOOOONG time. But do you think it would help?
CJ
[1]:
https://svn.apache.org/viewvc/httpd/httpd/trunk/modules/loggers/mod_log_json.c?view=markup&sortby=date
[2]:
http://people.apache.org/~jkaluza/patches/httpd-2.4.x-errorlog_provider.patch
Re: [RFC] JSON logging support for httpd 2.4.x
Posted by Thomas Meyer <th...@m3y3r.de>.
Hi,
Sadly I got no feedback at all.
What is the preferred way for contributions?
I also did raise a PR here with some fixes on top of this patch series:
https://github.com/apache/httpd/pull/353
Mfg
Thomas
Am 24. März 2023 22:54:08 MEZ schrieb Thomas Meyer <th...@m3y3r.de>:
>
>Hi,
>
>please have a look at this preliminarily work to support JSON output in mod_log_config.
>
>It's still unfinished and has probably a lot of bugs, but this is to show the general idea
>of my solution.
>
>Help and feedback is most welcome.
>
>Mfg
>thomas
>
>
--
Diese Nachricht wurde von meinem Android-Gerät mit K-9 Mail gesendet.
[PATCH 2/3] mod_log_config: Forward log_format_items to the log_writer
Posted by Thomas Meyer <th...@m3y3r.de>.
Prepare for JSON writer
---
modules/loggers/mod_log_config.c | 6 +++++-
modules/loggers/mod_log_config.h | 1 +
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/modules/loggers/mod_log_config.c b/modules/loggers/mod_log_config.c
index 6934a8691c..d142c888ad 100644
--- a/modules/loggers/mod_log_config.c
+++ b/modules/loggers/mod_log_config.c
@@ -185,12 +185,14 @@ static apr_status_t ap_default_log_writer(request_rec *r,
const char **strs,
int *strl,
int nelts,
+ void *items,
apr_size_t len);
static apr_status_t ap_buffered_log_writer(request_rec *r,
void *handle,
const char **strs,
int *strl,
int nelts,
+ void *items,
apr_size_t len);
static void *ap_default_log_writer_init(apr_pool_t *p, server_rec *s,
const char* name);
@@ -1178,7 +1180,7 @@ static int config_log_transaction(request_rec *r, config_log_state *cls,
"log writer isn't correctly setup");
return HTTP_INTERNAL_SERVER_ERROR;
}
- rv = log_writer(r, cls->log_writer, strs, strl, format->nelts, len);
+ rv = log_writer(r, cls->log_writer, strs, strl, format->nelts, items, len);
if (rv != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_WARNING, rv, r, APLOGNO(00646)
"Error writing to %s", cls->fname);
@@ -1611,6 +1613,7 @@ static apr_status_t ap_default_log_writer( request_rec *r,
const char **strs,
int *strl,
int nelts,
+ void *items,
apr_size_t len)
{
@@ -1684,6 +1687,7 @@ static apr_status_t ap_buffered_log_writer(request_rec *r,
const char **strs,
int *strl,
int nelts,
+ void *items,
apr_size_t len)
{
diff --git a/modules/loggers/mod_log_config.h b/modules/loggers/mod_log_config.h
index 877a593262..591362635d 100644
--- a/modules/loggers/mod_log_config.h
+++ b/modules/loggers/mod_log_config.h
@@ -49,6 +49,7 @@ typedef apr_status_t ap_log_writer(
const char **portions,
int *lengths,
int nelts,
+ void *items,
apr_size_t len);
typedef struct ap_log_handler {
--
2.20.1