You are viewing a plain text version of this content. The canonical link for it is here.
Posted to cvs@httpd.apache.org by yl...@apache.org on 2018/07/16 12:49:55 UTC
svn commit: r1836032 - in /httpd/httpd/trunk: include/ap_mmn.h
include/http_core.h server/core.c server/util_filter.c
Author: ylavic
Date: Mon Jul 16 12:49:55 2018
New Revision: 1836032
URL: http://svn.apache.org/viewvc?rev=1836032&view=rev
Log:
core: Add ReadBufferSize, FlushMaxThreshold and FlushMaxPipelined directives.
ReadBufferSize allows to configure the size of read buffers, for now it's
mainly used for file buckets reads (apr_bucket_file_set_buf_size), but it could
be used to replace AP_IOBUFSIZE in multiple places.
FlushMaxThreshold and FlushMaxPipelined allow to configure the hardcoded
THRESHOLD_MAX_BUFFER and MAX_REQUESTS_IN_PIPELINE from "util_filter.c".
The former sets the maximum size above which pending data are forcibly flushed
to the network (blocking eventually), and the latter sets the number of
pipelined/pending responses above which they are flushed regardless of whether
a pipelined request is immediately available (zero disables pipelining).
Larger ReadBufferSize and FlushMaxThreshold can trade memory consumption for
performances with the capacity of today's networks.
Modified:
httpd/httpd/trunk/include/ap_mmn.h
httpd/httpd/trunk/include/http_core.h
httpd/httpd/trunk/server/core.c
httpd/httpd/trunk/server/util_filter.c
Modified: httpd/httpd/trunk/include/ap_mmn.h
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/include/ap_mmn.h?rev=1836032&r1=1836031&r2=1836032&view=diff
==============================================================================
--- httpd/httpd/trunk/include/ap_mmn.h (original)
+++ httpd/httpd/trunk/include/ap_mmn.h Mon Jul 16 12:49:55 2018
@@ -582,6 +582,9 @@
* ap_filter_t
* 20180711.2 (2.5.1-dev) Add ap_reuse_brigade_from_pool()
* 20180716.1 (2.5.1-dev) Axe conn_rec->empty brigade
+ * 20180716.2 (2.5.1-dev) Add read_buf_size member to core_dir_config,
+ * flush_max_threshold and flush_max_pipelined to
+ * core_server_config, and ap_get_read_buf_size().
*/
#define MODULE_MAGIC_COOKIE 0x41503235UL /* "AP25" */
@@ -589,7 +592,7 @@
#ifndef MODULE_MAGIC_NUMBER_MAJOR
#define MODULE_MAGIC_NUMBER_MAJOR 20180716
#endif
-#define MODULE_MAGIC_NUMBER_MINOR 1 /* 0...n */
+#define MODULE_MAGIC_NUMBER_MINOR 2 /* 0...n */
/**
* Determine if the server's current MODULE_MAGIC_NUMBER is at least a
Modified: httpd/httpd/trunk/include/http_core.h
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/include/http_core.h?rev=1836032&r1=1836031&r2=1836032&view=diff
==============================================================================
--- httpd/httpd/trunk/include/http_core.h (original)
+++ httpd/httpd/trunk/include/http_core.h Mon Jul 16 12:49:55 2018
@@ -254,6 +254,13 @@ AP_DECLARE(const char *) ap_get_server_n
AP_DECLARE(apr_port_t) ap_get_server_port(const request_rec *r);
/**
+ * Get the size of read buffers
+ * @param r The current request
+ * @return The read buffers size
+ */
+AP_DECLARE(apr_size_t) ap_get_read_buf_size(const request_rec *r);
+
+/**
* Return the limit on bytes in request msg body
* @param r The current request
* @return the maximum number of bytes in the request msg body
@@ -676,6 +683,8 @@ typedef struct {
/** Table of rules for building CGI variables, NULL if none configured */
apr_hash_t *cgi_var_rules;
+
+ apr_size_t read_buf_size;
} core_dir_config;
/* macro to implement off by default behaviour */
@@ -758,6 +767,9 @@ typedef struct {
int protocols_honor_order;
int async_filter;
unsigned int async_filter_set:1;
+
+ apr_size_t flush_max_threshold;
+ apr_int32_t flush_max_pipelined;
} core_server_config;
/* for AddOutputFiltersByType in core.c */
Modified: httpd/httpd/trunk/server/core.c
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/server/core.c?rev=1836032&r1=1836031&r2=1836032&view=diff
==============================================================================
--- httpd/httpd/trunk/server/core.c (original)
+++ httpd/httpd/trunk/server/core.c Mon Jul 16 12:49:55 2018
@@ -30,6 +30,11 @@
#include "apr_random.h"
#endif
+#include "apr_version.h"
+#if APR_MAJOR_VERSION < 2
+#include "apu_version.h"
+#endif
+
#define APR_WANT_IOVEC
#define APR_WANT_STRFUNC
#define APR_WANT_MEMFUNC
@@ -91,6 +96,9 @@
#define AP_CONTENT_MD5_ON 1
#define AP_CONTENT_MD5_UNSET 2
+#define AP_FLUSH_MAX_THRESHOLD 65536
+#define AP_FLUSH_MAX_PIPELINED 5
+
APR_HOOK_STRUCT(
APR_HOOK_LINK(get_mgmt_items)
APR_HOOK_LINK(insert_network_bucket)
@@ -406,6 +414,13 @@ static void *merge_core_dir_configs(apr_
if (new->enable_sendfile != ENABLE_SENDFILE_UNSET) {
conf->enable_sendfile = new->enable_sendfile;
}
+
+ if (new->read_buf_size) {
+ conf->read_buf_size = new->read_buf_size;
+ }
+ else {
+ conf->read_buf_size = base->read_buf_size;
+ }
if (new->allow_encoded_slashes_set) {
conf->allow_encoded_slashes = new->allow_encoded_slashes;
@@ -482,14 +497,13 @@ static void *create_core_server_config(a
apr_table_setn(conf->accf_map, "http", "data");
apr_table_setn(conf->accf_map, "https", "data");
#endif
+
+ conf->flush_max_threshold = AP_FLUSH_MAX_THRESHOLD;
+ conf->flush_max_pipelined = AP_FLUSH_MAX_PIPELINED;
}
- /* pcalloc'ed - we have NULL's/0's
- else ** is_virtual ** {
- conf->ap_document_root = NULL;
- conf->access_name = NULL;
- conf->accf_map = NULL;
+ else {
+ conf->flush_max_pipelined = -1;
}
- */
/* initialization, no special case for global context */
@@ -593,11 +607,19 @@ static void *merge_core_server_configs(a
conf->protocols_honor_order = ((virt->protocols_honor_order < 0) ?
base->protocols_honor_order :
virt->protocols_honor_order);
+
conf->async_filter = ((virt->async_filter_set) ?
virt->async_filter :
base->async_filter);
conf->async_filter_set = base->async_filter_set || virt->async_filter_set;
+ conf->flush_max_threshold = (virt->flush_max_threshold)
+ ? virt->flush_max_threshold
+ : base->flush_max_threshold;
+ conf->flush_max_pipelined = (virt->flush_max_pipelined >= 0)
+ ? virt->flush_max_pipelined
+ : base->flush_max_pipelined;
+
return conf;
}
@@ -1259,6 +1281,13 @@ AP_DECLARE(apr_off_t) ap_get_limit_req_b
return d->limit_req_body;
}
+AP_DECLARE(apr_size_t) ap_get_read_buf_size(const request_rec *r)
+{
+ core_dir_config *d = ap_get_core_module_config(r->per_dir_config);
+
+ return d->read_buf_size ? d->read_buf_size : AP_IOBUFSIZE;
+}
+
/*****************************************************************
*
@@ -2285,6 +2314,65 @@ static const char *set_enable_sendfile(c
return NULL;
}
+static const char *set_read_buf_size(cmd_parms *cmd, void *d_,
+ const char *arg)
+{
+ core_dir_config *d = d_;
+ apr_off_t size;
+ char *end;
+
+ if (apr_strtoff(&size, arg, &end, 10)
+ || size < 0 || size > APR_SIZE_MAX || *end)
+ return apr_pstrcat(cmd->pool,
+ "parameter must be a number between 0 and "
+ APR_STRINGIFY(APR_SIZE_MAX) "): ",
+ arg, NULL);
+
+ d->read_buf_size = (apr_size_t)size;
+
+ return NULL;
+}
+
+static const char *set_flush_max_threshold(cmd_parms *cmd, void *d_,
+ const char *arg)
+{
+ core_server_config *conf =
+ ap_get_core_module_config(cmd->server->module_config);
+ apr_off_t size;
+ char *end;
+
+ if (apr_strtoff(&size, arg, &end, 10)
+ || size <= 0 || size > APR_SIZE_MAX || *end)
+ return apr_pstrcat(cmd->pool,
+ "parameter must be a number between 1 and "
+ APR_STRINGIFY(APR_SIZE_MAX) "): ",
+ arg, NULL);
+
+ conf->flush_max_threshold = (apr_size_t)size;
+
+ return NULL;
+}
+
+static const char *set_flush_max_pipelined(cmd_parms *cmd, void *d_,
+ const char *arg)
+{
+ core_server_config *conf =
+ ap_get_core_module_config(cmd->server->module_config);
+ apr_off_t num;
+ char *end;
+
+ if (apr_strtoff(&num, arg, &end, 10)
+ || num < 0 || num > APR_INT32_MAX || *end)
+ return apr_pstrcat(cmd->pool,
+ "parameter must be a number between 0 and "
+ APR_STRINGIFY(APR_INT32_MAX) ": ",
+ arg, NULL);
+
+ conf->flush_max_pipelined = (apr_int32_t)num;
+
+ return NULL;
+}
+
/*
* Report a missing-'>' syntax error.
@@ -4594,6 +4682,12 @@ AP_INIT_TAKE1("EnableMMAP", set_enable_m
"Controls whether memory-mapping may be used to read files"),
AP_INIT_TAKE1("EnableSendfile", set_enable_sendfile, NULL, OR_FILEINFO,
"Controls whether sendfile may be used to transmit files"),
+AP_INIT_TAKE1("ReadBufferSize", set_read_buf_size, NULL, OR_FILEINFO,
+ "Size (in bytes) of the memory buffers used to read data"),
+AP_INIT_TAKE1("FlushMaxThreshold", set_flush_max_threshold, NULL, RSRC_CONF,
+ "Maximum size (in bytes) above which pending data are flushed (blocking) to the network"),
+AP_INIT_TAKE1("FlushMaxPipelined", set_flush_max_pipelined, NULL, RSRC_CONF,
+ "Number of pipelined/pending responses above which they are flushed to the network"),
/* Old server config file commands */
@@ -5044,6 +5138,11 @@ static int default_handler(request_rec *
(void)apr_bucket_file_enable_mmap(e, 0);
}
#endif
+#if APR_MAJOR_VERSION > 1 || (APU_MAJOR_VERSION == 1 && APU_MINOR_VERSION >= 6)
+ if (d->read_buf_size) {
+ apr_bucket_file_set_buf_size(e, d->read_buf_size);
+ }
+#endif
}
e = apr_bucket_eos_create(c->bucket_alloc);
Modified: httpd/httpd/trunk/server/util_filter.c
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/server/util_filter.c?rev=1836032&r1=1836031&r2=1836032&view=diff
==============================================================================
--- httpd/httpd/trunk/server/util_filter.c (original)
+++ httpd/httpd/trunk/server/util_filter.c Mon Jul 16 12:49:55 2018
@@ -33,10 +33,6 @@
#define FILTER_POOL apr_hook_global_pool
#include "ap_hooks.h" /* for apr_hook_global_pool */
-/* XXX: Should these be configurable parameters? */
-#define THRESHOLD_MAX_BUFFER 65536
-#define MAX_REQUESTS_IN_PIPELINE 5
-
/*
** This macro returns true/false if a given filter should be inserted BEFORE
** another filter. This will happen when one of: 1) there isn't another
@@ -825,7 +821,8 @@ AP_DECLARE(apr_status_t) ap_filter_reins
apr_size_t bytes_in_brigade, non_file_bytes_in_brigade;
int eor_buckets_in_brigade, morphing_bucket_in_brigade;
int loglevel = ap_get_conn_module_loglevel(f->c, APLOG_MODULE_INDEX);
-
+ core_server_config *conf;
+
if (loglevel >= APLOG_TRACE6) {
ap_log_cerror(
APLOG_MARK, APLOG_TRACE6, 0, f->c,
@@ -845,16 +842,16 @@ AP_DECLARE(apr_status_t) ap_filter_reins
* of everything up that point.
*
* b) The request is in CONN_STATE_HANDLER state, and the brigade
- * contains at least THRESHOLD_MAX_BUFFER bytes in non-file
+ * contains at least flush_max_threshold bytes in non-file
* buckets: Do blocking writes until the amount of data in the
- * buffer is less than THRESHOLD_MAX_BUFFER. (The point of this
+ * buffer is less than flush_max_threshold. (The point of this
* rule is to provide flow control, in case a handler is
* streaming out lots of data faster than the data can be
* sent to the client.)
*
* c) The request is in CONN_STATE_HANDLER state, and the brigade
- * contains at least MAX_REQUESTS_IN_PIPELINE EOR buckets:
- * Do blocking writes until less than MAX_REQUESTS_IN_PIPELINE EOR
+ * contains at least flush_max_pipelined EOR buckets:
+ * Do blocking writes until less than flush_max_pipelined EOR
* buckets are left. (The point of this rule is to prevent too many
* FDs being kept open by pipelined requests, possibly allowing a
* DoS).
@@ -862,7 +859,7 @@ AP_DECLARE(apr_status_t) ap_filter_reins
* d) The request is being served by a connection filter and the
* brigade contains a morphing bucket: If there was no other
* reason to do a blocking write yet, try reading the bucket. If its
- * contents fit into memory before THRESHOLD_MAX_BUFFER is reached,
+ * contents fit into memory before flush_max_threshold is reached,
* everything is fine. Otherwise we need to do a blocking write the
* up to and including the morphing bucket, because ap_save_brigade()
* would read the whole bucket into memory later on.
@@ -875,6 +872,8 @@ AP_DECLARE(apr_status_t) ap_filter_reins
eor_buckets_in_brigade = 0;
morphing_bucket_in_brigade = 0;
+ conf = ap_get_core_module_config(f->c->base_server->module_config);
+
for (bucket = APR_BRIGADE_FIRST(bb); bucket != APR_BRIGADE_SENTINEL(bb);
bucket = next) {
next = APR_BUCKET_NEXT(bucket);
@@ -899,18 +898,18 @@ AP_DECLARE(apr_status_t) ap_filter_reins
}
if (APR_BUCKET_IS_FLUSH(bucket)
- || non_file_bytes_in_brigade >= THRESHOLD_MAX_BUFFER
+ || non_file_bytes_in_brigade >= conf->flush_max_threshold
|| (!f->r && morphing_bucket_in_brigade)
- || eor_buckets_in_brigade > MAX_REQUESTS_IN_PIPELINE) {
+ || eor_buckets_in_brigade > conf->flush_max_pipelined) {
/* this segment of the brigade MUST be sent before returning. */
if (loglevel >= APLOG_TRACE6) {
char *reason = APR_BUCKET_IS_FLUSH(bucket) ?
"FLUSH bucket" :
- (non_file_bytes_in_brigade >= THRESHOLD_MAX_BUFFER) ?
- "THRESHOLD_MAX_BUFFER" :
+ (non_file_bytes_in_brigade >= conf->flush_max_threshold) ?
+ "max threshold" :
(!f->r && morphing_bucket_in_brigade) ? "morphing bucket" :
- "MAX_REQUESTS_IN_PIPELINE";
+ "max requests in pipeline";
ap_log_cerror(APLOG_MARK, APLOG_TRACE6, 0, f->c,
"will flush because of %s", reason);
ap_log_cerror(APLOG_MARK, APLOG_TRACE8, 0, f->c,