You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by sh...@apache.org on 2019/09/03 13:56:38 UTC
[trafficserver] branch master updated (5ec8112 -> 8e4776c)
This is an automated email from the ASF dual-hosted git repository.
shinrich pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git.
from 5ec8112 Provide stats for the recently introduced HTTP/2 rate limits
new 45592cf pipe buffer size for log.pipe should be configurable
new 8e4776c address review comments
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
doc/admin-guide/logging/destinations.en.rst | 3 +++
doc/admin-guide/logging/examples.en.rst | 20 ++++++++++++++++++
proxy/logging/LogFile.cc | 32 +++++++++++++++++++++++++++--
proxy/logging/LogFile.h | 3 ++-
proxy/logging/LogObject.cc | 12 +++++++----
proxy/logging/LogObject.h | 5 ++++-
proxy/logging/YamlLogConfig.cc | 23 +++++++++++++++------
7 files changed, 84 insertions(+), 14 deletions(-)
[trafficserver] 02/02: address review comments
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shinrich pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 8e4776c24e58e9bf41a6040a3aad71fd64e3a1ef
Author: Eric Schwartz <es...@gmail.com>
AuthorDate: Tue Aug 27 20:50:40 2019 +0000
address review comments
---
doc/admin-guide/logging/destinations.en.rst | 3 +++
doc/admin-guide/logging/examples.en.rst | 20 ++++++++++++++++++++
proxy/logging/LogFile.cc | 4 ++++
3 files changed, 27 insertions(+)
diff --git a/doc/admin-guide/logging/destinations.en.rst b/doc/admin-guide/logging/destinations.en.rst
index 5b70913..03b6f75 100644
--- a/doc/admin-guide/logging/destinations.en.rst
+++ b/doc/admin-guide/logging/destinations.en.rst
@@ -109,6 +109,9 @@ pipe, only full records are dropped.
Output to named pipes is always, as the mode's name implies, in ASCII format.
There is no option for logging binary format log data to a named pipe.
+For ASCII pipes there exists an option to set the ``pipe_buffer_size`` in
+the YAML config.
+
.. _admin-logging-ascii-v-binary:
Deciding Between ASCII or Binary Output
diff --git a/doc/admin-guide/logging/examples.en.rst b/doc/admin-guide/logging/examples.en.rst
index 4cca17e..f1d0840 100644
--- a/doc/admin-guide/logging/examples.en.rst
+++ b/doc/admin-guide/logging/examples.en.rst
@@ -299,6 +299,26 @@ for them to a UNIX pipe that the alerting software can constantly read from.
- canaryfilter
filename: alerting_canaries
+Configuring ASCII Pipe Buffer Size
+==================================
+
+This example mirrors the one above but also sets a ```pipe_buffer_size``` of
+1024 * 1024 for the pipe. This can be set on a per-pipe basis but is not
+available on FreeBSD dists of ATS. If this field is not set, the pipe buffer
+will default to the OS default size.
+
+.. code:: yaml
+
+ logs:
+ - mode: pipe
+ format: canaryformat
+ filters:
+ - canaryfilter
+ filename: alerting_canaries
+ pipe_buffer_size: 1048576
+
+
+
Summarizing Origin Responses by Hour
====================================
diff --git a/proxy/logging/LogFile.cc b/proxy/logging/LogFile.cc
index c9e911c..4154329 100644
--- a/proxy/logging/LogFile.cc
+++ b/proxy/logging/LogFile.cc
@@ -188,6 +188,9 @@ LogFile::open_file()
return LOG_FILE_NO_PIPE_READERS;
}
+#ifdef __FreeBSD__
+ // we don't do this for FreeBSD
+#else
// adjust pipe size if necessary
if (m_pipe_buffer_size) {
long pipe_size = (long)fcntl(m_fd, F_GETPIPE_SZ);
@@ -209,6 +212,7 @@ LogFile::open_file()
Debug("log-file", "NEW pipe size for pipe %s = %ld", m_name, pipe_size);
}
}
+#endif
} else {
if (m_log) {
int status = m_log->open_file(Log::config->logfile_perm);
[trafficserver] 01/02: pipe buffer size for log.pipe should be
configurable
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shinrich pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 45592cff2f8e424f87c80059a62befdbfdd06633
Author: Eric Schwartz <es...@gmail.com>
AuthorDate: Thu Aug 22 14:36:44 2019 +0000
pipe buffer size for log.pipe should be configurable
Conflicts:
proxy/logging/YamlLogConfig.cc
---
proxy/logging/LogFile.cc | 28 ++++++++++++++++++++++++++--
proxy/logging/LogFile.h | 3 ++-
proxy/logging/LogObject.cc | 12 ++++++++----
proxy/logging/LogObject.h | 5 ++++-
proxy/logging/YamlLogConfig.cc | 23 +++++++++++++++++------
5 files changed, 57 insertions(+), 14 deletions(-)
diff --git a/proxy/logging/LogFile.cc b/proxy/logging/LogFile.cc
index c0928be..c9e911c 100644
--- a/proxy/logging/LogFile.cc
+++ b/proxy/logging/LogFile.cc
@@ -63,12 +63,13 @@
-------------------------------------------------------------------------*/
LogFile::LogFile(const char *name, const char *header, LogFileFormat format, uint64_t signature, size_t ascii_buffer_size,
- size_t max_line_size)
+ size_t max_line_size, int pipe_buffer_size)
: m_file_format(format),
m_name(ats_strdup(name)),
m_header(ats_strdup(header)),
m_signature(signature),
- m_max_line_size(max_line_size)
+ m_max_line_size(max_line_size),
+ m_pipe_buffer_size(pipe_buffer_size)
{
if (m_file_format != LOG_FILE_PIPE) {
m_log = new BaseLogFile(name, m_signature);
@@ -97,6 +98,7 @@ LogFile::LogFile(const LogFile ©)
m_signature(copy.m_signature),
m_ascii_buffer_size(copy.m_ascii_buffer_size),
m_max_line_size(copy.m_max_line_size),
+ m_pipe_buffer_size(copy.m_pipe_buffer_size),
m_fd(copy.m_fd)
{
ink_release_assert(m_ascii_buffer_size >= m_max_line_size);
@@ -185,6 +187,28 @@ LogFile::open_file()
Debug("log-file", "no readers for pipe %s", m_name);
return LOG_FILE_NO_PIPE_READERS;
}
+
+ // adjust pipe size if necessary
+ if (m_pipe_buffer_size) {
+ long pipe_size = (long)fcntl(m_fd, F_GETPIPE_SZ);
+ if (pipe_size == -1) {
+ Error("get pipe size failed for pipe %s", m_name);
+ } else {
+ Debug("log-file", "Default pipe size for pipe %s = %ld", m_name, pipe_size);
+ }
+
+ int ret = fcntl(m_fd, F_SETPIPE_SZ, m_pipe_buffer_size);
+ if (ret == -1) {
+ Error("set pipe size failed for pipe %s", m_name);
+ }
+
+ pipe_size = (long)fcntl(m_fd, F_GETPIPE_SZ);
+ if (pipe_size == -1) {
+ Error("get pipe size failed for pipe %s", m_name);
+ } else {
+ Debug("log-file", "NEW pipe size for pipe %s = %ld", m_name, pipe_size);
+ }
+ }
} else {
if (m_log) {
int status = m_log->open_file(Log::config->logfile_perm);
diff --git a/proxy/logging/LogFile.h b/proxy/logging/LogFile.h
index 4f2fd65..1a9a7f1 100644
--- a/proxy/logging/LogFile.h
+++ b/proxy/logging/LogFile.h
@@ -43,7 +43,7 @@ class LogFile : public LogBufferSink, public RefCountObj
{
public:
LogFile(const char *name, const char *header, LogFileFormat format, uint64_t signature, size_t ascii_buffer_size = 4 * 9216,
- size_t max_line_size = 9216);
+ size_t max_line_size = 9216, int pipe_buffer_size = 0);
LogFile(const LogFile &);
~LogFile() override;
@@ -120,6 +120,7 @@ public:
uint64_t m_signature; // signature of log object stored
size_t m_ascii_buffer_size; // size of ascii buffer
size_t m_max_line_size; // size of longest log line (record)
+ int m_pipe_buffer_size; // this is the size of the pipe buffer set by fcntl
int m_fd; // this could back m_log or a pipe, depending on the situation
public:
diff --git a/proxy/logging/LogObject.cc b/proxy/logging/LogObject.cc
index 8032dbd..f1c2337 100644
--- a/proxy/logging/LogObject.cc
+++ b/proxy/logging/LogObject.cc
@@ -90,7 +90,8 @@ LogBufferManager::preproc_buffers(LogBufferSink *sink)
LogObject::LogObject(const LogFormat *format, const char *log_dir, const char *basename, LogFileFormat file_format,
const char *header, Log::RollingEnabledValues rolling_enabled, int flush_threads, int rolling_interval_sec,
- int rolling_offset_hr, int rolling_size_mb, bool auto_created, int max_rolled, bool reopen_after_rolling)
+ int rolling_offset_hr, int rolling_size_mb, bool auto_created, int max_rolled, bool reopen_after_rolling,
+ int pipe_buffer_size)
: m_alt_filename(nullptr),
m_flags(0),
m_signature(0),
@@ -101,7 +102,8 @@ LogObject::LogObject(const LogFormat *format, const char *log_dir, const char *b
m_last_roll_time(0),
m_max_rolled(max_rolled),
m_reopen_after_rolling(reopen_after_rolling),
- m_buffer_manager_idx(0)
+ m_buffer_manager_idx(0),
+ m_pipe_buffer_size(pipe_buffer_size)
{
ink_release_assert(format);
m_format = new LogFormat(*format);
@@ -118,7 +120,8 @@ LogObject::LogObject(const LogFormat *format, const char *log_dir, const char *b
// compute_signature is a static function
m_signature = compute_signature(m_format, m_basename, m_flags);
- m_logFile = new LogFile(m_filename, header, file_format, m_signature, Log::config->ascii_buffer_size, Log::config->max_line_size);
+ m_logFile = new LogFile(m_filename, header, file_format, m_signature, Log::config->ascii_buffer_size, Log::config->max_line_size,
+ m_pipe_buffer_size);
if (m_reopen_after_rolling) {
m_logFile->open_file();
@@ -148,7 +151,8 @@ LogObject::LogObject(LogObject &rhs)
m_last_roll_time(rhs.m_last_roll_time),
m_max_rolled(rhs.m_max_rolled),
m_reopen_after_rolling(rhs.m_reopen_after_rolling),
- m_buffer_manager_idx(rhs.m_buffer_manager_idx)
+ m_buffer_manager_idx(rhs.m_buffer_manager_idx),
+ m_pipe_buffer_size(rhs.m_pipe_buffer_size)
{
m_format = new LogFormat(*(rhs.m_format));
m_buffer_manager = new LogBufferManager[m_flush_threads];
diff --git a/proxy/logging/LogObject.h b/proxy/logging/LogObject.h
index fcf6046..32dafb1 100644
--- a/proxy/logging/LogObject.h
+++ b/proxy/logging/LogObject.h
@@ -95,7 +95,8 @@ public:
LogObject(const LogFormat *format, const char *log_dir, const char *basename, LogFileFormat file_format, const char *header,
Log::RollingEnabledValues rolling_enabled, int flush_threads, int rolling_interval_sec = 0, int rolling_offset_hr = 0,
- int rolling_size_mb = 0, bool auto_created = false, int rolling_max_count = 0, bool reopen_after_rolling = false);
+ int rolling_size_mb = 0, bool auto_created = false, int rolling_max_count = 0, bool reopen_after_rolling = false,
+ int pipe_buffer_size = 0);
LogObject(LogObject &);
~LogObject() override;
@@ -285,6 +286,8 @@ private:
unsigned m_buffer_manager_idx;
LogBufferManager *m_buffer_manager;
+ int m_pipe_buffer_size;
+
void generate_filenames(const char *log_dir, const char *basename, LogFileFormat file_format);
void _setup_rolling(Log::RollingEnabledValues rolling_enabled, int rolling_interval_sec, int rolling_offset_hr,
int rolling_size_mb);
diff --git a/proxy/logging/YamlLogConfig.cc b/proxy/logging/YamlLogConfig.cc
index 398bfcb..1011445 100644
--- a/proxy/logging/YamlLogConfig.cc
+++ b/proxy/logging/YamlLogConfig.cc
@@ -111,7 +111,8 @@ TsEnumDescriptor ROLLING_MODE_LUA = {
std::set<std::string> valid_log_object_keys = {
"filename", "format", "mode", "header", "rolling_enabled", "rolling_interval_sec",
- "rolling_offset_hr", "rolling_size_mb", "filters", "min_count", "rolling_max_count", "rolling_allow_empty"};
+ "rolling_offset_hr", "rolling_size_mb", "filters", "min_count", "rolling_max_count", "rolling_allow_empty",
+ "pipe_buffer_size"};
LogObject *
YamlLogConfig::decodeLogObject(const YAML::Node &node)
@@ -196,11 +197,21 @@ YamlLogConfig::decodeLogObject(const YAML::Node &node)
Warning("Invalid log rolling value '%d' in log object", obj_rolling_enabled);
}
- auto logObject =
- new LogObject(fmt, Log::config->logfile_dir, filename.c_str(), file_type, header.c_str(),
- static_cast<Log::RollingEnabledValues>(obj_rolling_enabled), Log::config->preproc_threads,
- obj_rolling_interval_sec, obj_rolling_offset_hr, obj_rolling_size_mb, /* auto_created */ false,
- /* rolling_max_count */ obj_rolling_max_count, /* reopen_after_rolling */ obj_rolling_allow_empty > 0);
+ // get buffer for pipe
+ int pipe_buffer_size = 0;
+ if (node["pipe_buffer_size"]) {
+ if (file_type != LOG_FILE_PIPE) {
+ Warning("Pipe buffer size field should only be set for log.pipe object.");
+ } else {
+ pipe_buffer_size = node["pipe_buffer_size"].as<int>();
+ }
+ }
+
+ auto logObject = new LogObject(fmt, Log::config->logfile_dir, filename.c_str(), file_type, header.c_str(),
+ static_cast<Log::RollingEnabledValues>(obj_rolling_enabled), Log::config->preproc_threads,
+ obj_rolling_interval_sec, obj_rolling_offset_hr, obj_rolling_size_mb, /* auto_created */ false,
+ /* rolling_max_count */ obj_rolling_max_count,
+ /* reopen_after_rolling */ obj_rolling_allow_empty > 0, pipe_buffer_size);
// Generate LogDeletingInfo entry for later use
std::string ext;