You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by ma...@apache.org on 2022/08/15 23:27:50 UTC
[trafficserver] branch 10-Dev updated: Add log field names pqu* and deprecates cqu* (#9019)
This is an automated email from the ASF dual-hosted git repository.
maskit pushed a commit to branch 10-Dev
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/10-Dev by this push:
new ec291ab82 Add log field names pqu* and deprecates cqu* (#9019)
ec291ab82 is described below
commit ec291ab82fc563076763479d2be2ecd8ea53565b
Author: Masakazu Kitajo <ma...@apache.org>
AuthorDate: Tue Aug 16 08:27:45 2022 +0900
Add log field names pqu* and deprecates cqu* (#9019)
* Add log field names pqu* and deprecated cqu*
* Update logstats.blog
---
configs/logging.yaml.default | 6 ++---
doc/admin-guide/layer-4-routing.en.rst | 4 +--
doc/admin-guide/logging/examples.en.rst | 6 ++---
doc/admin-guide/logging/formatting.en.rst | 33 +++++++++++++++++++------
doc/appendices/faq.en.rst | 2 +-
proxy/logging/Log.cc | 20 +++++++++++++++
proxy/logging/LogAccess.cc | 2 +-
src/traffic_logstats/logstats.cc | 2 +-
src/traffic_logstats/tests/logstats.blog | Bin 9144 -> 9144 bytes
tests/gold_tests/ip_allow/ip_allow.test.py | 4 +--
tests/gold_tests/logging/log-filenames.test.py | 2 +-
tests/gold_tests/logging/sigusr2.test.py | 2 +-
tests/gold_tests/redirect/redirect.test.py | 2 +-
tests/gold_tests/timeout/conn_timeout.test.py | 2 +-
tests/gold_tests/tls/tls_client_cert.test.py | 2 +-
tests/gold_tests/tls/tls_client_cert2.test.py | 2 +-
tests/gold_tests/tls/tls_client_verify.test.py | 2 +-
17 files changed, 65 insertions(+), 28 deletions(-)
diff --git a/configs/logging.yaml.default b/configs/logging.yaml.default
index a29e65379..0e8aebb01 100644
--- a/configs/logging.yaml.default
+++ b/configs/logging.yaml.default
@@ -19,16 +19,16 @@ logging:
# reporting tools, simply create a log that uses this format.
- name: welf
format: |-
- id=firewall time="%<cqtd> %<cqtt>" fw=%<phn> pri=6 proto=%<cqus> duration=%<ttmsf> sent=%<psql> rcvd=%<cqhl> src=%<chi> dst=%<shi> dstname=%<shn> user=%<caun> op=%<cqhm> arg="%<cqup>" result=%<pssc> ref="%<{Referer}cqh>" agent="%<{user-agent}cqh>" cache=%<crc>
+ id=firewall time="%<cqtd> %<cqtt>" fw=%<phn> pri=6 proto=%<pqus> duration=%<ttmsf> sent=%<psql> rcvd=%<cqhl> src=%<chi> dst=%<shi> dstname=%<shn> user=%<caun> op=%<cqhm> arg="%<pqup>" result=%<pssc> ref="%<{Referer}cqh>" agent="%<{user-agent}cqh>" cache=%<crc>
# Squid Log Format with seconds resolution timestamp.
# The following is the squid format but with a seconds-only timestamp
# (cqts) instead of a seconds and milliseconds timestamp (cqtq).
- name: squid_seconds_only_timestamp
- format: '%<cqts> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<caun> %<phr>/%<shn> %<psct>'
+ format: '%<cqts> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<pquc> %<caun> %<phr>/%<shn> %<psct>'
# Squid Log Format.
- name: squid
- format: '%<cqtq> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<caun> %<phr>/%<shn> %<psct>'
+ format: '%<cqtq> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<pquc> %<caun> %<phr>/%<shn> %<psct>'
# Common Log Format.
- name: common
diff --git a/doc/admin-guide/layer-4-routing.en.rst b/doc/admin-guide/layer-4-routing.en.rst
index 2485a43ab..463d94762 100644
--- a/doc/admin-guide/layer-4-routing.en.rst
+++ b/doc/admin-guide/layer-4-routing.en.rst
@@ -62,8 +62,8 @@ request to itself with the ``tunnel_route`` host and port as the upstream. That
connection is treated as if the user agent had sent a
``CONNECT`` to the upstream and forwards the "`CLIENT HELLO
<https://tools.ietf.org/html/rfc5246#section-7.4.1.2>`__" to it. In addition to the method appearing
-as ``CONNECT``, be aware that logs printing the URL via the ``<%cquc>`` field format will show the
-scheme in the URL as ``tunnel``. The scheme as printed via ``<%cqus>``, however, will show the
+as ``CONNECT``, be aware that logs printing the URL via the ``<%pquc>`` field format will show the
+scheme in the URL as ``tunnel``. The scheme as printed via ``<%pqus>``, however, will show the
scheme used in the original client request.
Example
diff --git a/doc/admin-guide/logging/examples.en.rst b/doc/admin-guide/logging/examples.en.rst
index 9d2009231..1671697a0 100644
--- a/doc/admin-guide/logging/examples.en.rst
+++ b/doc/admin-guide/logging/examples.en.rst
@@ -212,7 +212,7 @@ following format object:
formats:
- name: squid
- format: '%<cqtq> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<caun> %<phr>/%<shn> %<psct>'
+ format: '%<cqtq> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<pquc> %<caun> %<phr>/%<shn> %<psct>'
Hourly Rotated Squid Proxy Logs
===============================
@@ -225,7 +225,7 @@ policy.
formats:
- name: squid
- format: '%<cqtq> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<caun> %<phr>/%<shn> %<psct>'
+ format: '%<cqtq> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<pquc> %<caun> %<phr>/%<shn> %<psct>'
logs:
- mode: ascii
@@ -290,7 +290,7 @@ for them to a UNIX pipe that the alerting software can constantly read from.
filters:
- name: canaryfilter
- accept: cqup MATCH "/nightmare/scenario/dont/touch"
+ accept: pqup MATCH "/nightmare/scenario/dont/touch"
logs:
- mode: ascii_pipe
diff --git a/doc/admin-guide/logging/formatting.en.rst b/doc/admin-guide/logging/formatting.en.rst
index e1ba9f3e5..2f0f76de4 100644
--- a/doc/admin-guide/logging/formatting.en.rst
+++ b/doc/admin-guide/logging/formatting.en.rst
@@ -808,6 +808,10 @@ URLs, Schemes, and Paths
.. _cquc:
.. _cqup:
.. _cqus:
+.. _pqu:
+.. _pquc:
+.. _pqup:
+.. _pqus:
.. _cquuc:
.. _cquup:
.. _cquuh:
@@ -821,13 +825,26 @@ Field Source Description
cqu Proxy Request URL of the proxy request from |TS| to the origin. If a
remap rule is used (:file:`remap.config`), the original
client URL is replaced with the URL in the target of the
- remap rule.
+ remap rule. Deprecated since 10.0. Use ``pqu`` instead.
cquc Proxy Request Canonical URL from the proxy request to origin. This field
differs from cqu_ by having its contents URL-escaped
(spaces and various other characters are replaced by
+ percent-escaped entity codes). Deprecated since 10.0.
+ Use ``pquc`` instead.
+cqup Proxy Request Path component from the proxy request. Deprecated since
+ 10.0. Use ``pqup`` instead.
+cqus Proxy Request URL scheme from the proxy request. Deprecated since 10.0.
+ Use ``pqus`` instead.
+pqu Proxy Request URL of the proxy request from |TS| to the origin. If a
+ remap rule is used (:file:`remap.config`), the original
+ client URL is replaced with the URL in the target of the
+ remap rule.
+pquc Proxy Request Canonical URL from the proxy request to origin. This field
+ differs from pqu_ by having its contents URL-escaped
+ (spaces and various other characters are replaced by
percent-escaped entity codes).
-cqup Proxy Request Path component from the proxy request.
-cqus Proxy Request URL scheme from the proxy request.
+pqup Proxy Request Path component from the proxy request.
+pqus Proxy Request URL scheme from the proxy request.
cquuc Client Request Canonical (prior to remapping) effective URL from client request.
It is possible that plugins prior to the remap phase have
changed the client request, so this field may not match the
@@ -877,11 +894,11 @@ The below slice specifiers are allowed.
Some examples below ::
- '%<cqup>' // the whole characters of <cqup>.
- '%<cqup>[:]' // the whole characters of <cqup>.
- '%<cqup[0:30]>' // the first 30 characters of <cqup>.
- '%<cqup[-10:]>' // the last 10 characters of <cqup>.
- '%<cqup[:-5]>' // everything except the last 5 characters of <cqup>.
+ '%<pqup>' // the whole characters of <pqup>.
+ '%<pqup>[:]' // the whole characters of <pqup>.
+ '%<pqup[0:30]>' // the first 30 characters of <pqup>.
+ '%<pqup[-10:]>' // the last 10 characters of <pqup>.
+ '%<pqup[:-5]>' // everything except the last 5 characters of <pqup>.
Note when ``escape`` in ``format`` is set to ``json``, the start is the
position before escaping JSON strings, and escaped values are sliced at
diff --git a/doc/appendices/faq.en.rst b/doc/appendices/faq.en.rst
index bdb861c0d..ea248734f 100644
--- a/doc/appendices/faq.en.rst
+++ b/doc/appendices/faq.en.rst
@@ -508,7 +508,7 @@ This happens when a log entry is too big to fit in a log buffer. This is often
URLs in the entry. You can truncate long URLs using the slice syntax for the URL log field in the log
format, for example::
- %<cquc[:1000]>
+ %<pquc[:1000]>
You can also increase the value of :ts:cv:`proxy.config.log.log_buffer_size`, but this can have impacts
on performance and memory usage. Very large values may trigger software bugs. Some production proxies
diff --git a/proxy/logging/Log.cc b/proxy/logging/Log.cc
index 58d19facf..bd1d84918 100644
--- a/proxy/logging/Log.cc
+++ b/proxy/logging/Log.cc
@@ -403,11 +403,21 @@ Log::init_fields()
global_field_list.add(field, false);
field_symbol_hash.emplace("cqu", field);
+ field = new LogField("client_req_url", "pqu", LogField::STRING, &LogAccess::marshal_client_req_url, &LogAccess::unmarshal_str,
+ &LogAccess::set_client_req_url);
+ global_field_list.add(field, false);
+ field_symbol_hash.emplace("pqu", field);
+
field = new LogField("client_req_url_canonical", "cquc", LogField::STRING, &LogAccess::marshal_client_req_url_canon,
&LogAccess::unmarshal_str, &LogAccess::set_client_req_url_canon);
global_field_list.add(field, false);
field_symbol_hash.emplace("cquc", field);
+ field = new LogField("client_req_url_canonical", "pquc", LogField::STRING, &LogAccess::marshal_client_req_url_canon,
+ &LogAccess::unmarshal_str, &LogAccess::set_client_req_url_canon);
+ global_field_list.add(field, false);
+ field_symbol_hash.emplace("pquc", field);
+
field =
new LogField("client_req_unmapped_url_canonical", "cquuc", LogField::STRING, &LogAccess::marshal_client_req_unmapped_url_canon,
&LogAccess::unmarshal_str, &LogAccess::set_client_req_unmapped_url_canon);
@@ -429,11 +439,21 @@ Log::init_fields()
global_field_list.add(field, false);
field_symbol_hash.emplace("cqus", field);
+ field = new LogField("client_req_url_scheme", "pqus", LogField::STRING, &LogAccess::marshal_client_req_url_scheme,
+ &LogAccess::unmarshal_str);
+ global_field_list.add(field, false);
+ field_symbol_hash.emplace("pqus", field);
+
field = new LogField("client_req_url_path", "cqup", LogField::STRING, &LogAccess::marshal_client_req_url_path,
&LogAccess::unmarshal_str, &LogAccess::set_client_req_url_path);
global_field_list.add(field, false);
field_symbol_hash.emplace("cqup", field);
+ field = new LogField("client_req_url_path", "pqup", LogField::STRING, &LogAccess::marshal_client_req_url_path,
+ &LogAccess::unmarshal_str, &LogAccess::set_client_req_url_path);
+ global_field_list.add(field, false);
+ field_symbol_hash.emplace("pqup", field);
+
field = new LogField("client_req_http_version", "cqhv", LogField::dINT, &LogAccess::marshal_client_req_http_version,
&LogAccess::unmarshal_http_version);
global_field_list.add(field, false);
diff --git a/proxy/logging/LogAccess.cc b/proxy/logging/LogAccess.cc
index 6116688b3..9f5f50d64 100644
--- a/proxy/logging/LogAccess.cc
+++ b/proxy/logging/LogAccess.cc
@@ -916,7 +916,7 @@ LogAccess::unmarshal_http_version(char **buf, char *dest, int len)
/*-------------------------------------------------------------------------
LogAccess::unmarshal_http_text
- The http text is simply the fields http_method (cqhm) + url (cqu) +
+ The http text is simply the fields http_method (cqhm) + url (pqu) +
http_version (cqhv), all right next to each other, in that order.
-------------------------------------------------------------------------*/
diff --git a/src/traffic_logstats/logstats.cc b/src/traffic_logstats/logstats.cc
index 75ab708fa..c9c151c9f 100644
--- a/src/traffic_logstats/logstats.cc
+++ b/src/traffic_logstats/logstats.cc
@@ -1280,7 +1280,7 @@ parse_log_buff(LogBufferHeader *buf_header, bool summary = false, bool aggregate
if (!cl.no_format_check) {
// Validate the fieldlist
field = fieldlist->first();
- const std::string_view test_fields[] = {"cqtq", "ttms", "chi", "crc", "pssc", "psql", "cqhm", "cquc", "caun", "phr", "shn"};
+ const std::string_view test_fields[] = {"cqtq", "ttms", "chi", "crc", "pssc", "psql", "cqhm", "pquc", "caun", "phr", "shn"};
for (auto i : test_fields) {
if (i != field->symbol()) {
cerr << "Error parsing log file - expected field: " << i << ", but read field: " << field->symbol() << endl;
diff --git a/src/traffic_logstats/tests/logstats.blog b/src/traffic_logstats/tests/logstats.blog
index 72c09f733..b36e76648 100644
Binary files a/src/traffic_logstats/tests/logstats.blog and b/src/traffic_logstats/tests/logstats.blog differ
diff --git a/tests/gold_tests/ip_allow/ip_allow.test.py b/tests/gold_tests/ip_allow/ip_allow.test.py
index 5edebe617..784023cd2 100644
--- a/tests/gold_tests/ip_allow/ip_allow.test.py
+++ b/tests/gold_tests/ip_allow/ip_allow.test.py
@@ -95,11 +95,11 @@ ts.Disk.records_config.update({
})
format_string = ('%<cqtd>-%<cqtt> %<stms> %<ttms> %<chi> %<crc>/%<pssc> %<psql> '
- '%<cqhm> %<cquc> %<phr>/%<pqsn> %<psct> %<{Y-RID}pqh> '
+ '%<cqhm> %<pquc> %<phr>/%<pqsn> %<psct> %<{Y-RID}pqh> '
'%<{Y-YPCS}pqh> %<{Host}cqh> %<{CHAD}pqh> '
'sftover=%<{x-safet-overlimit-rules}cqh> sftmat=%<{x-safet-matched-rules}cqh> '
'sftcls=%<{x-safet-classification}cqh> '
- 'sftbadclf=%<{x-safet-bad-classifiers}cqh> yra=%<{Y-RA}cqh> scheme=%<cqus>')
+ 'sftbadclf=%<{x-safet-bad-classifiers}cqh> yra=%<{Y-RA}cqh> scheme=%<pqus>')
ts.Disk.logging_yaml.AddLines(
''' logging:
diff --git a/tests/gold_tests/logging/log-filenames.test.py b/tests/gold_tests/logging/log-filenames.test.py
index e94068ada..d3913d23a 100644
--- a/tests/gold_tests/logging/log-filenames.test.py
+++ b/tests/gold_tests/logging/log-filenames.test.py
@@ -88,7 +88,7 @@ class LogFilenamesTest:
logging:
formats:
- name: url_and_return_code
- format: "%<cqu>: %<pssc>"
+ format: "%<pqu>: %<pssc>"
logs:
- filename: {self.sentinel_log_filename}
format: url_and_return_code
diff --git a/tests/gold_tests/logging/sigusr2.test.py b/tests/gold_tests/logging/sigusr2.test.py
index f644e766d..a93b4eeb5 100644
--- a/tests/gold_tests/logging/sigusr2.test.py
+++ b/tests/gold_tests/logging/sigusr2.test.py
@@ -66,7 +66,7 @@ class Sigusr2Test:
logging:
formats:
- name: has_path
- format: "%<cqu>: %<sssc>"
+ format: "%<pqu>: %<sssc>"
logs:
- filename: test_rotation
format: has_path
diff --git a/tests/gold_tests/redirect/redirect.test.py b/tests/gold_tests/redirect/redirect.test.py
index 01777dc37..3074ab706 100644
--- a/tests/gold_tests/redirect/redirect.test.py
+++ b/tests/gold_tests/redirect/redirect.test.py
@@ -46,7 +46,7 @@ ts.Disk.logging_yaml.AddLines(
logging:
formats:
- name: custom
- format: "client_url=%<cqu> cache_result: code=%<crc> subcode=%<crsc>"
+ format: "client_url=%<pqu> cache_result: code=%<crc> subcode=%<crsc>"
logs:
- filename: the_log
format: custom
diff --git a/tests/gold_tests/timeout/conn_timeout.test.py b/tests/gold_tests/timeout/conn_timeout.test.py
index 7eb34cf71..0a5b64e46 100644
--- a/tests/gold_tests/timeout/conn_timeout.test.py
+++ b/tests/gold_tests/timeout/conn_timeout.test.py
@@ -47,7 +47,7 @@ ts.Disk.logging_yaml.AddLines(
logging:
formats:
- name: testformat
- format: '%<pssc> %<cquc> %<pscert> %<cscert>'
+ format: '%<pssc> %<pquc> %<pscert> %<cscert>'
logs:
- mode: ascii
format: testformat
diff --git a/tests/gold_tests/tls/tls_client_cert.test.py b/tests/gold_tests/tls/tls_client_cert.test.py
index 04569c88d..baebf84f9 100644
--- a/tests/gold_tests/tls/tls_client_cert.test.py
+++ b/tests/gold_tests/tls/tls_client_cert.test.py
@@ -109,7 +109,7 @@ ts.Disk.logging_yaml.AddLines(
logging:
formats:
- name: testformat
- format: '%<pssc> %<cquc> %<pscert> %<cscert>'
+ format: '%<pssc> %<pquc> %<pscert> %<cscert>'
logs:
- mode: ascii
format: testformat
diff --git a/tests/gold_tests/tls/tls_client_cert2.test.py b/tests/gold_tests/tls/tls_client_cert2.test.py
index 207fdac32..a028b409f 100644
--- a/tests/gold_tests/tls/tls_client_cert2.test.py
+++ b/tests/gold_tests/tls/tls_client_cert2.test.py
@@ -110,7 +110,7 @@ ts.Disk.logging_yaml.AddLines(
logging:
formats:
- name: testformat
- format: '%<pssc> %<cquc> %<pscert> %<cscert>'
+ format: '%<pssc> %<pquc> %<pscert> %<cscert>'
logs:
- mode: ascii
format: testformat
diff --git a/tests/gold_tests/tls/tls_client_verify.test.py b/tests/gold_tests/tls/tls_client_verify.test.py
index cda82eeb8..de962278a 100644
--- a/tests/gold_tests/tls/tls_client_verify.test.py
+++ b/tests/gold_tests/tls/tls_client_verify.test.py
@@ -76,7 +76,7 @@ ts.Disk.logging_yaml.AddLines(
logging:
formats:
- name: testformat
- format: '%<pssc> %<cquc> %<pscert> %<cscert>'
+ format: '%<pssc> %<pquc> %<pscert> %<cscert>'
logs:
- mode: ascii
format: testformat