You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by ma...@apache.org on 2021/10/04 02:21:59 UTC
[trafficserver] branch master updated: Pre-warming TLS Tunnel
(#7661)
This is an automated email from the ASF dual-hosted git repository.
masaori pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new d1e2dd8 Pre-warming TLS Tunnel (#7661)
d1e2dd8 is described below
commit d1e2dd8aac6a1c2cd340418b9534923413d1650c
Author: Masaori Koshiba <ma...@apache.org>
AuthorDate: Mon Oct 4 11:21:47 2021 +0900
Pre-warming TLS Tunnel (#7661)
---
.gitignore | 1 +
doc/admin-guide/files/records.config.en.rst | 24 +
doc/admin-guide/files/sni.yaml.en.rst | 21 +
doc/admin-guide/layer-4-routing.en.rst | 48 +
.../monitoring/statistics/core/ssl.en.rst | 42 +
doc/uml/l4-pre-warming-overview.uml | 32 +
iocore/eventsystem/I_EThread.h | 3 +
iocore/eventsystem/I_Thread.h | 1 +
iocore/net/P_SNIActionPerformer.h | 12 +-
iocore/net/P_SSLNetVConnection.h | 23 +-
iocore/net/SSLSNIConfig.cc | 8 +-
iocore/net/YamlSNIConfig.cc | 64 +-
iocore/net/YamlSNIConfig.h | 16 +
iocore/net/libinknet_stub.cc | 10 +
mgmt/RecordsConfig.cc | 8 +
proxy/http/HttpConfig.h | 2 +
proxy/http/HttpProxyServerMain.cc | 3 +
proxy/http/HttpSM.cc | 54 +-
proxy/http/HttpSM.h | 2 +
proxy/http/Makefile.am | 16 +-
proxy/http/PreWarmAlgorithm.h | 122 ++
proxy/http/PreWarmConfig.cc | 75 ++
proxy/http/PreWarmConfig.h | 57 +
proxy/http/PreWarmManager.cc | 1170 ++++++++++++++++++++
proxy/http/PreWarmManager.h | 341 ++++++
proxy/http/unit_tests/test_PreWarm.cc | 223 ++++
src/traffic_quic/traffic_quic.cc | 8 +
27 files changed, 2367 insertions(+), 19 deletions(-)
diff --git a/.gitignore b/.gitignore
index 801e96e..0de9256 100644
--- a/.gitignore
+++ b/.gitignore
@@ -116,6 +116,7 @@ proxy/http/remap/test_PluginDso
proxy/http/remap/test_PluginFactory
proxy/http/remap/test_RemapPluginInfo
proxy/http/test_proxy_http
+proxy/http/test_PreWarm
proxy/http/remap/test_*
proxy/http2/test_libhttp2
proxy/http2/test_Http2DependencyTree
diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst
index b73c834..03e9e29 100644
--- a/doc/admin-guide/files/records.config.en.rst
+++ b/doc/admin-guide/files/records.config.en.rst
@@ -3949,6 +3949,30 @@ SNI Routing
Frequency of checking the activity of SNI Routing Tunnel. Set to ``0`` to disable monitoring of the activity of the SNI tunnels.
The feature is disabled by default.
+.. ts:cv:: CONFIG proxy.config.tunnel.prewarm INT 0
+
+ Enable :ref:`pre-warming-tls-tunnel`. The feature is disabled by default.
+
+.. ts:cv:: CONFIG proxy.config.tunnel.prewarm.max_stats_size INT 100
+
+ Max size of :ref:`dynamic stats for Pre-warming TLS Tunnel <pre-warming-tls-tunnel-stats>`.
+
+.. ts:cv:: CONFIG proxy.config.tunnel.prewarm.algorithm INT 2
+
+ Version of pre-warming algorithm.
+
+ ===== ======================================================================
+ Value Description
+ ===== ======================================================================
+ ``1`` Periodical pre-warming only
+ ``2`` Event based pre-warming + Periodical pre-warming
+ ===== ======================================================================
+
+.. ts:cv:: CONFIG proxy.config.tunnel.prewarm.event_period INT 1000
+ :units: milliseconds
+
+ Frequency of periodical pre-warming in milli-seconds.
+
OCSP Stapling Configuration
===========================
diff --git a/doc/admin-guide/files/sni.yaml.en.rst b/doc/admin-guide/files/sni.yaml.en.rst
index c1ff0a6..7a04e4b 100644
--- a/doc/admin-guide/files/sni.yaml.en.rst
+++ b/doc/admin-guide/files/sni.yaml.en.rst
@@ -159,6 +159,27 @@ tunnel_alpn Inbound List of ALPN Protocol Ids for Partial Blind
This only works with ``partial_blind_route``.
========================= ========= ========================================================================================
+Pre-warming TLS Tunnel
+----------------------
+
+=============================== ========================================================================================
+Key Meaning
+=============================== ========================================================================================
+tunnel_prewarm Override :ts:cv:`proxy.config.tunnel.prewarm` in records.config.
+
+tunnel_prewarm_srv Enable SRV record lookup on pre-warming. Default is ``false``.
+
+tunnel_prewarm_rate Rate of how many connections to pre-warm. Default is ``1.0``.
+
+tunnel_prewarm_min Minimum number of pre-warming queue size (per thread). Default is ``0``.
+
+tunnel_prewarm_max Maximum number of pre-warming queue size (per thread). Default is ``-1`` (unlimited).
+
+tunnel_prewarm_connect_timeout Timeout for TCP/TLS handshake (in seconds).
+
+tunnel_prewarm_inactive_timeout Inactive timeout for connections in the pool (in seconds).
+=============================== ========================================================================================
+
Client verification, via ``verify_client``, corresponds to setting
:ts:cv:`proxy.config.ssl.client.certification_level` for this connection as noted below.
diff --git a/doc/admin-guide/layer-4-routing.en.rst b/doc/admin-guide/layer-4-routing.en.rst
index b652897..2485a43 100644
--- a/doc/admin-guide/layer-4-routing.en.rst
+++ b/doc/admin-guide/layer-4-routing.en.rst
@@ -129,3 +129,51 @@ tunneled connection like this, the only transaction hooks that will be triggered
:c:macro:`TS_HTTP_TXN_START_HOOK` and :c:macro:`TS_HTTP_TXN_CLOSE_HOOK`. In addition, because |TS|
does not terminate (and therefore does not decrypt) the connection, it cannot be cached or served from
cache.
+
+.. _pre-warming-tls-tunnel:
+
+Pre-warming TLS Tunnel
+======================
+
+Pre-warming TLS Tunnel reduces the latency of TLS connections (``forward_route`` and ``partial_blind_route`` type SNI
+Routing). When this feature is enabled, each ET_NET thread makes TLS connections pool per routing type, SNI, and ALPN.
+
+.. figure:: ../uml/images/l4-pre-warming-overview.svg
+ :align: center
+
+Stats for connection pools are registered dynamically on start up. Details in :ref:`pre-warming-tls-tunnel-stats`.
+
+Examples
+--------
+
+.. code:: yaml
+
+ sni:
+ - fqdn: foo.com
+ http2: off
+ partial_blind_route: bar.com
+ client_sni_policy: server_name
+ tunnel_prewarm: true
+ tunnel_prewarm_connect_timeout: 10
+ tunnel_prewarm_inactive_timeout: 150
+ tunnel_prewarm_max: 100
+ tunnel_prewarm_min: 10
+ tunnel_alpn:
+ - h2
+
+.. code::
+
+ proxy.process.tunnel.prewarm.bar.com:443.tls.current_init 0
+ proxy.process.tunnel.prewarm.bar.com:443.tls.current_open 10
+ proxy.process.tunnel.prewarm.bar.com:443.tls.total_hit 0
+ proxy.process.tunnel.prewarm.bar.com:443.tls.total_miss 0
+ proxy.process.tunnel.prewarm.bar.com:443.tls.total_handshake_time 1106250000
+ proxy.process.tunnel.prewarm.bar.com:443.tls.total_handshake_count 10
+ proxy.process.tunnel.prewarm.bar.com:443.tls.total_retry 0
+ proxy.process.tunnel.prewarm.bar.com:443.tls.http2.current_init 0
+ proxy.process.tunnel.prewarm.bar.com:443.tls.http2.current_open 10
+ proxy.process.tunnel.prewarm.bar.com:443.tls.http2.total_hit 0
+ proxy.process.tunnel.prewarm.bar.com:443.tls.http2.total_miss 0
+ proxy.process.tunnel.prewarm.bar.com:443.tls.http2.total_handshake_time 1142368000
+ proxy.process.tunnel.prewarm.bar.com:443.tls.http2.total_handshake_count 10
+ proxy.process.tunnel.prewarm.bar.com:443.tls.http2.total_retry 0
diff --git a/doc/admin-guide/monitoring/statistics/core/ssl.en.rst b/doc/admin-guide/monitoring/statistics/core/ssl.en.rst
index 4f51c6c..e18bae1 100644
--- a/doc/admin-guide/monitoring/statistics/core/ssl.en.rst
+++ b/doc/admin-guide/monitoring/statistics/core/ssl.en.rst
@@ -237,3 +237,45 @@ SSL/TLS
:type: gauge
A gauge of current active SNI Routing Tunnels.
+
+.. _pre-warming-tls-tunnel-stats:
+
+Pre-warming TLS Tunnel
+----------------------
+
+Stats for Pre-warming TLS Tunnel is registered dynamically. The ``POOL`` in below represents combination of ``<Hostname of destination>.<Type of Tunnel>.<ALPN Name (if there)>``.
+
+.. ts:stat:: global proxy.process.tunnel.prewarm.POOL.current_init integer
+ :type: gauge
+
+ Represents the current number of initializing connections in the pool.
+
+.. ts:stat:: global proxy.process.tunnel.prewarm.POOL.current_open integer
+ :type: gauge
+
+ Represents the current number of established connections in the pool.
+
+.. ts:stat:: global proxy.process.tunnel.prewarm.POOL.total_hit integer
+ :type: counter
+
+ Represents the total number of pre-warmed connection is used.
+
+.. ts:stat:: global proxy.process.tunnel.prewarm.POOL.total_miss integer
+ :type: counter
+
+ Represents the total number of pre-warmed connection is not used.
+
+.. ts:stat:: global proxy.process.tunnel.prewarm.POOL.total_handshake_time integer
+ :type: counter
+
+ Represents the total number of handshake duration of pre-warming.
+
+.. ts:stat:: global proxy.process.tunnel.prewarm.POOL.total_handshake_count integer
+ :type: counter
+
+ Represents the total number of handshake time of pre-warming.
+
+.. ts:stat:: global proxy.process.tunnel.prewarm.POOL.total_retry integer
+ :type: counter
+
+ Represents the total number of pre-warming retry.
diff --git a/doc/uml/l4-pre-warming-overview.uml b/doc/uml/l4-pre-warming-overview.uml
new file mode 100644
index 0000000..5367de0
--- /dev/null
+++ b/doc/uml/l4-pre-warming-overview.uml
@@ -0,0 +1,32 @@
+' Licensed under the Apache License, Version 2.0 (the "License");
+' you may not use this file except in compliance with the License.
+' You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+' Unless required by applicable law or agreed to in writing, software distributed under the License is distributed
+' on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+' See the License for the specific language governing permissions and limitations under the License.
+
+@startuml
+'title Pre-warm TLS Tunnel
+skinparam sequenceMessageAlign direction
+skinparam ParticipantPadding 75
+'skinparam monochrome reverse
+skinparam backgroundColor #white
+hide footbox
+
+participant client
+participant proxy
+participant origin_server
+
+group pre-warm connection pool
+ proxy -> origin_server : open connection
+ proxy -> origin_server : open connection
+ proxy -> origin_server : open connection
+ ...
+end
+...
+group pool size > 0
+ client -> proxy : open connection
+ hnote over proxy #white: use pre-warmed connection from pool
+ rnote over client, origin_server #lightgreen: TLS Partial Blind Tunnel
+end
+@enduml
diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h
index 49d74d8..8da21e6 100644
--- a/iocore/eventsystem/I_EThread.h
+++ b/iocore/eventsystem/I_EThread.h
@@ -43,6 +43,8 @@ struct DiskHandler;
struct EventIO;
class ServerSessionPool;
+class PreWarmQueue;
+
class Event;
class Continuation;
@@ -349,6 +351,7 @@ public:
Event *start_event = nullptr;
ServerSessionPool *server_session_pool = nullptr;
+ PreWarmQueue *prewarm_queue = nullptr;
/** Default handler used until it is overridden.
diff --git a/iocore/eventsystem/I_Thread.h b/iocore/eventsystem/I_Thread.h
index 8037d2a..432d14a 100644
--- a/iocore/eventsystem/I_Thread.h
+++ b/iocore/eventsystem/I_Thread.h
@@ -135,6 +135,7 @@ public:
ProxyAllocator ioDataAllocator;
ProxyAllocator ioAllocator;
ProxyAllocator ioBlockAllocator;
+ ProxyAllocator preWarmSMAllocator;
// From InkAPI (plugins wrappers)
ProxyAllocator apiHookAllocator;
ProxyAllocator INKContAllocator;
diff --git a/iocore/net/P_SNIActionPerformer.h b/iocore/net/P_SNIActionPerformer.h
index 11e2691..7fc7924 100644
--- a/iocore/net/P_SNIActionPerformer.h
+++ b/iocore/net/P_SNIActionPerformer.h
@@ -98,8 +98,9 @@ private:
class TunnelDestination : public ActionItem
{
public:
- TunnelDestination(const std::string_view &dest, SNIRoutingType type, const std::vector<int> &alpn)
- : destination(dest), type(type), alpn_ids(alpn)
+ TunnelDestination(const std::string_view &dest, SNIRoutingType type, YamlSNIConfig::TunnelPreWarm prewarm,
+ const std::vector<int> &alpn)
+ : destination(dest), type(type), tunnel_prewarm(prewarm), alpn_ids(alpn)
{
need_fix = (destination.find_first_of('$') != std::string::npos);
}
@@ -115,10 +116,10 @@ public:
// If needed, we will try to amend the tunnel destination.
if (ctx._fqdn_wildcard_captured_groups && need_fix) {
const auto &fixed_dst = replace_match_groups(destination, *ctx._fqdn_wildcard_captured_groups);
- ssl_netvc->set_tunnel_destination(fixed_dst, type);
+ ssl_netvc->set_tunnel_destination(fixed_dst, type, tunnel_prewarm);
Debug("ssl_sni", "Destination now is [%s], configured [%s], fqdn [%s]", fixed_dst.c_str(), destination.c_str(), servername);
} else {
- ssl_netvc->set_tunnel_destination(destination, type);
+ ssl_netvc->set_tunnel_destination(destination, type, tunnel_prewarm);
Debug("ssl_sni", "Destination now is [%s], fqdn [%s]", destination.c_str(), servername);
}
@@ -202,7 +203,8 @@ private:
}
std::string destination;
- SNIRoutingType type = SNIRoutingType::NONE;
+ SNIRoutingType type = SNIRoutingType::NONE;
+ YamlSNIConfig::TunnelPreWarm tunnel_prewarm = YamlSNIConfig::TunnelPreWarm::UNSET;
const std::vector<int> &alpn_ids;
bool need_fix;
};
diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h
index d389552..1691c0e 100644
--- a/iocore/net/P_SSLNetVConnection.h
+++ b/iocore/net/P_SSLNetVConnection.h
@@ -298,11 +298,13 @@ public:
bool decrypt_tunnel() const;
bool upstream_tls() const;
SNIRoutingType tunnel_type() const;
+ YamlSNIConfig::TunnelPreWarm tunnel_prewarm() const;
void
- set_tunnel_destination(const std::string_view &destination, SNIRoutingType type)
+ set_tunnel_destination(const std::string_view &destination, SNIRoutingType type, YamlSNIConfig::TunnelPreWarm prewarm)
{
- _tunnel_type = type;
+ _tunnel_type = type;
+ _tunnel_prewarm = prewarm;
auto pos = destination.find(":");
if (nullptr != tunnel_host) {
@@ -486,10 +488,13 @@ private:
HANDSHAKE_HOOKS_DONE
} sslHandshakeHookState = HANDSHAKE_HOOKS_PRE;
- int64_t redoWriteSize = 0;
- char *tunnel_host = nullptr;
- in_port_t tunnel_port = 0;
- SNIRoutingType _tunnel_type = SNIRoutingType::NONE;
+ int64_t redoWriteSize = 0;
+
+ char *tunnel_host = nullptr;
+ in_port_t tunnel_port = 0;
+ SNIRoutingType _tunnel_type = SNIRoutingType::NONE;
+ YamlSNIConfig::TunnelPreWarm _tunnel_prewarm = YamlSNIConfig::TunnelPreWarm::UNSET;
+
X509_STORE_CTX *verify_cert = nullptr;
// Null-terminated string, or nullptr if there is no SNI server name.
@@ -523,6 +528,12 @@ SSLNetVConnection::tunnel_type() const
return _tunnel_type;
}
+inline YamlSNIConfig::TunnelPreWarm
+SSLNetVConnection::tunnel_prewarm() const
+{
+ return _tunnel_prewarm;
+}
+
/**
Returns true if this vc was configured for forward_route or partial_blind_route
*/
diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc
index ae82275..c134c82 100644
--- a/iocore/net/SSLSNIConfig.cc
+++ b/iocore/net/SSLSNIConfig.cc
@@ -30,6 +30,9 @@
****************************************************************************/
#include "P_SSLSNI.h"
+
+#include "PreWarmManager.h"
+
#include "tscore/Diags.h"
#include "tscore/SimpleTokenizer.h"
#include "tscore/ink_memory.h"
@@ -77,7 +80,8 @@ SNIConfigParams::loadSNIConfig()
ai->actions.push_back(std::make_unique<TLSValidProtocols>(item.protocol_mask));
}
if (item.tunnel_destination.length() > 0) {
- ai->actions.push_back(std::make_unique<TunnelDestination>(item.tunnel_destination, item.tunnel_type, item.tunnel_alpn));
+ ai->actions.push_back(
+ std::make_unique<TunnelDestination>(item.tunnel_destination, item.tunnel_type, item.tunnel_prewarm, item.tunnel_alpn));
}
if (!item.client_sni_policy.empty()) {
ai->actions.push_back(std::make_unique<OutboundSNIPolicy>(item.client_sni_policy));
@@ -200,6 +204,8 @@ SNIConfig::reconfigure()
params->Initialize();
configid = configProcessor.set(configid, params);
+
+ prewarmManager.reconfigure();
}
SNIConfigParams *
diff --git a/iocore/net/YamlSNIConfig.cc b/iocore/net/YamlSNIConfig.cc
index 2142e84..cadeb77 100644
--- a/iocore/net/YamlSNIConfig.cc
+++ b/iocore/net/YamlSNIConfig.cc
@@ -130,6 +130,13 @@ std::set<std::string> valid_sni_config_keys = {TS_fqdn,
TS_forward_route,
TS_partial_blind_route,
TS_tunnel_alpn,
+ TS_tunnel_prewarm,
+ TS_tunnel_prewarm_min,
+ TS_tunnel_prewarm_max,
+ TS_tunnel_prewarm_rate,
+ TS_tunnel_prewarm_connect_timeout,
+ TS_tunnel_prewarm_inactive_timeout,
+ TS_tunnel_prewarm_srv,
TS_verify_server_policy,
TS_verify_server_properties,
TS_client_cert,
@@ -232,15 +239,64 @@ template <> struct convert<YamlSNIConfig::Item> {
item.host_sni_policy = static_cast<uint8_t>(policy);
}
+ YamlSNIConfig::TunnelPreWarm t_prewarm = YamlSNIConfig::TunnelPreWarm::UNSET;
+ uint32_t t_min = item.tunnel_prewarm_min;
+ int32_t t_max = item.tunnel_prewarm_max;
+ double t_rate = item.tunnel_prewarm_rate;
+ uint32_t t_connect_timeout = item.tunnel_prewarm_connect_timeout;
+ uint32_t t_inactive_timeout = item.tunnel_prewarm_inactive_timeout;
+ bool t_srv = item.tunnel_prewarm_srv;
+
+ if (node[TS_tunnel_prewarm]) {
+ auto is_prewarm_enabled = node[TS_tunnel_prewarm].as<bool>();
+ if (is_prewarm_enabled) {
+ t_prewarm = YamlSNIConfig::TunnelPreWarm::ENABLED;
+ } else {
+ t_prewarm = YamlSNIConfig::TunnelPreWarm::DISABLED;
+ }
+ }
+ if (node[TS_tunnel_prewarm_min]) {
+ t_min = node[TS_tunnel_prewarm_min].as<uint32_t>();
+ }
+ if (node[TS_tunnel_prewarm_max]) {
+ t_max = node[TS_tunnel_prewarm_max].as<int32_t>();
+ }
+ if (node[TS_tunnel_prewarm_rate]) {
+ t_rate = node[TS_tunnel_prewarm_rate].as<double>();
+ }
+ if (node[TS_tunnel_prewarm_connect_timeout]) {
+ t_connect_timeout = node[TS_tunnel_prewarm_connect_timeout].as<uint32_t>();
+ }
+ if (node[TS_tunnel_prewarm_inactive_timeout]) {
+ t_inactive_timeout = node[TS_tunnel_prewarm_inactive_timeout].as<uint32_t>();
+ }
+ if (node[TS_tunnel_prewarm_srv]) {
+ t_srv = node[TS_tunnel_prewarm_srv].as<bool>();
+ }
+
if (node[TS_tunnel_route]) {
item.tunnel_destination = node[TS_tunnel_route].as<std::string>();
item.tunnel_type = SNIRoutingType::BLIND;
} else if (node[TS_forward_route]) {
- item.tunnel_destination = node[TS_forward_route].as<std::string>();
- item.tunnel_type = SNIRoutingType::FORWARD;
+ item.tunnel_destination = node[TS_forward_route].as<std::string>();
+ item.tunnel_type = SNIRoutingType::FORWARD;
+ item.tunnel_prewarm = t_prewarm;
+ item.tunnel_prewarm_min = t_min;
+ item.tunnel_prewarm_max = t_max;
+ item.tunnel_prewarm_rate = t_rate;
+ item.tunnel_prewarm_connect_timeout = t_connect_timeout;
+ item.tunnel_prewarm_inactive_timeout = t_inactive_timeout;
+ item.tunnel_prewarm_srv = t_srv;
} else if (node[TS_partial_blind_route]) {
- item.tunnel_destination = node[TS_partial_blind_route].as<std::string>();
- item.tunnel_type = SNIRoutingType::PARTIAL_BLIND;
+ item.tunnel_destination = node[TS_partial_blind_route].as<std::string>();
+ item.tunnel_type = SNIRoutingType::PARTIAL_BLIND;
+ item.tunnel_prewarm = t_prewarm;
+ item.tunnel_prewarm_min = t_min;
+ item.tunnel_prewarm_max = t_max;
+ item.tunnel_prewarm_rate = t_rate;
+ item.tunnel_prewarm_connect_timeout = t_connect_timeout;
+ item.tunnel_prewarm_inactive_timeout = t_inactive_timeout;
+ item.tunnel_prewarm_srv = t_srv;
if (node[TS_tunnel_alpn]) {
load_tunnel_alpn(item.tunnel_alpn, node[TS_tunnel_alpn]);
diff --git a/iocore/net/YamlSNIConfig.h b/iocore/net/YamlSNIConfig.h
index 5b10ff3..700e740 100644
--- a/iocore/net/YamlSNIConfig.h
+++ b/iocore/net/YamlSNIConfig.h
@@ -39,6 +39,13 @@ TSDECL(tunnel_route);
TSDECL(forward_route);
TSDECL(partial_blind_route);
TSDECL(tunnel_alpn);
+TSDECL(tunnel_prewarm);
+TSDECL(tunnel_prewarm_min);
+TSDECL(tunnel_prewarm_max);
+TSDECL(tunnel_prewarm_rate);
+TSDECL(tunnel_prewarm_connect_timeout);
+TSDECL(tunnel_prewarm_inactive_timeout);
+TSDECL(tunnel_prewarm_srv);
TSDECL(verify_server_policy);
TSDECL(verify_server_properties);
TSDECL(verify_origin_server);
@@ -55,6 +62,7 @@ struct YamlSNIConfig {
enum class Policy : uint8_t { DISABLED = 0, PERMISSIVE, ENFORCED, UNSET };
enum class Property : uint8_t { NONE = 0, SIGNATURE_MASK = 0x1, NAME_MASK = 0x2, ALL_MASK = 0x3, UNSET };
enum class TLSProtocol : uint8_t { TLSv1 = 0, TLSv1_1, TLSv1_2, TLSv1_3, TLS_MAX = TLSv1_3 };
+ enum class TunnelPreWarm : uint8_t { DISABLED = 0, ENABLED, UNSET };
YamlSNIConfig() {}
@@ -77,6 +85,14 @@ struct YamlSNIConfig {
unsigned long protocol_mask;
std::vector<int> tunnel_alpn{};
+ bool tunnel_prewarm_srv = false;
+ uint32_t tunnel_prewarm_min = 0;
+ int32_t tunnel_prewarm_max = -1;
+ double tunnel_prewarm_rate = 1.0;
+ uint32_t tunnel_prewarm_connect_timeout = 0;
+ uint32_t tunnel_prewarm_inactive_timeout = 0;
+ TunnelPreWarm tunnel_prewarm = TunnelPreWarm::UNSET;
+
void EnableProtocol(YamlSNIConfig::TLSProtocol proto);
};
diff --git a/iocore/net/libinknet_stub.cc b/iocore/net/libinknet_stub.cc
index 74091db..1c65bd3 100644
--- a/iocore/net/libinknet_stub.cc
+++ b/iocore/net/libinknet_stub.cc
@@ -175,3 +175,13 @@ ProcessManager::signalManager(int, char const *)
ink_assert(false);
return;
}
+
+#include "PreWarmManager.h"
+void
+PreWarmManager::reconfigure()
+{
+ ink_assert(false);
+ return;
+}
+
+PreWarmManager prewarmManager;
diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc
index fd00ca0..997d5fc 100644
--- a/mgmt/RecordsConfig.cc
+++ b/mgmt/RecordsConfig.cc
@@ -987,6 +987,14 @@ static const RecordElement RecordsConfig[] =
//##########################################################################
{RECT_CONFIG, "proxy.config.tunnel.activity_check_period", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-100]", RECA_NULL}
,
+ {RECT_CONFIG, "proxy.config.tunnel.prewarm", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-1]", RECA_NULL}
+ ,
+ {RECT_CONFIG, "proxy.config.tunnel.prewarm.max_stats_size", RECD_INT, "100", RECU_RESTART_TS, RR_NULL, RECC_INT, "[5-65536]", RECA_NULL}
+ ,
+ {RECT_CONFIG, "proxy.config.tunnel.prewarm.event_period", RECD_INT, "1000", RECU_DYNAMIC, RR_NULL, RECC_INT, "[10-3600000]", RECA_NULL}
+ ,
+ {RECT_CONFIG, "proxy.config.tunnel.prewarm.algorithm", RECD_INT, "2", RECU_DYNAMIC, RR_NULL, RECC_INT, "[1-2]", RECA_NULL}
+ ,
//##########################################################################
//#
diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h
index 368e331..233f875 100644
--- a/proxy/http/HttpConfig.h
+++ b/proxy/http/HttpConfig.h
@@ -864,6 +864,8 @@ public:
class HttpConfig
{
public:
+ using scoped_config = ConfigProcessor::scoped_config<HttpConfig, HttpConfigParams>;
+
static void startup();
static void reconfigure();
diff --git a/proxy/http/HttpProxyServerMain.cc b/proxy/http/HttpProxyServerMain.cc
index 1a38a8b..f5f1c3b 100644
--- a/proxy/http/HttpProxyServerMain.cc
+++ b/proxy/http/HttpProxyServerMain.cc
@@ -44,6 +44,7 @@
#include "P_QUICNextProtocolAccept.h"
#include "http3/Http3SessionAccept.h"
#endif
+#include "PreWarmManager.h"
#include <vector>
@@ -383,6 +384,8 @@ start_HttpProxyServer()
hook->invoke(TS_EVENT_LIFECYCLE_PORTS_READY, nullptr);
hook = hook->next();
}
+
+ prewarmManager.start();
}
void
diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc
index a11236c..ec77453 100644
--- a/proxy/http/HttpSM.cc
+++ b/proxy/http/HttpSM.cc
@@ -32,6 +32,8 @@
#include "HttpSessionManager.h"
#include "P_Cache.h"
#include "P_Net.h"
+#include "PreWarmConfig.h"
+#include "PreWarmManager.h"
#include "StatPages.h"
#include "Log.h"
#include "LogAccess.h"
@@ -378,6 +380,12 @@ HttpSM::cleanup()
transform_cache_sm.mutex.clear();
magic = HTTP_SM_MAGIC_DEAD;
debug_on = false;
+
+ if (_prewarm_sm) {
+ _prewarm_sm->destroy();
+ THREAD_FREE(_prewarm_sm, preWarmSMAllocator, this_ethread());
+ _prewarm_sm = nullptr;
+ }
}
void
@@ -5312,13 +5320,51 @@ HttpSM::do_http_server_open(bool raw)
if (ssl_vc && raw) {
tls_upstream = ssl_vc->upstream_tls();
_tunnel_type = ssl_vc->tunnel_type();
+
// ALPN on TLS Partial Blind Tunnel - set negotiated ALPN id
+ int pid = SessionProtocolNameRegistry::INVALID;
if (ssl_vc->tunnel_type() == SNIRoutingType::PARTIAL_BLIND) {
- int pid = ssl_vc->get_negotiated_protocol_id();
+ pid = ssl_vc->get_negotiated_protocol_id();
if (pid != SessionProtocolNameRegistry::INVALID) {
opt.alpn_protos = SessionProtocolNameRegistry::convert_openssl_alpn_wire_format(pid);
}
}
+
+ //
+ // Grab pre-warmed NetVConnection if possible
+ //
+ PreWarmConfig::scoped_config prewarm_conf;
+ bool use_prewarm = prewarm_conf->enabled;
+
+ // override "proxy.config.tunnel.prewarm" by "tunnel_prewarm" in sni.yaml
+ if (YamlSNIConfig::TunnelPreWarm sni_use_prewarm = ssl_vc->tunnel_prewarm();
+ sni_use_prewarm != YamlSNIConfig::TunnelPreWarm::UNSET) {
+ use_prewarm = static_cast<bool>(sni_use_prewarm);
+ }
+
+ if (use_prewarm) {
+ // TODO: avoid copy of string -> make map key std::variant
+ PreWarm::SPtrConstDst dst =
+ std::make_shared<const PreWarm::Dst>(ssl_vc->get_tunnel_host(), ssl_vc->get_tunnel_port(),
+ tls_upstream ? SNIRoutingType::PARTIAL_BLIND : SNIRoutingType::FORWARD, pid);
+
+ EThread *ethread = this_ethread();
+ _prewarm_sm = ethread->prewarm_queue->dequeue(dst);
+
+ if (_prewarm_sm != nullptr) {
+ NetVConnection *netvc = _prewarm_sm->move_netvc();
+ ink_release_assert(_prewarm_sm->handler == &PreWarmSM::state_closed);
+
+ SMDebug("http_ss", "using pre-warmed tunnel netvc=%p", netvc);
+
+ t_state.current.attempts = 0;
+
+ ink_release_assert(default_handler == HttpSM::default_handler);
+ handleEvent(NET_EVENT_OPEN, netvc);
+ return;
+ }
+ SMDebug("http_ss", "no pre-warmed tunnel");
+ }
}
opt.local_port = ua_txn->get_outbound_port();
@@ -7012,6 +7058,12 @@ HttpSM::setup_blind_tunnel(bool send_response_hdr, IOBufferReader *initial)
ua_raw_buffer_reader = nullptr;
}
+ // if pre-warmed connection is used and it has data from origin server, foward it to ua
+ if (_prewarm_sm && _prewarm_sm->has_data_from_origin_server()) {
+ ink_release_assert(_prewarm_sm->handler == &PreWarmSM::state_closed);
+ client_response_hdr_bytes += to_ua_buf->write(_prewarm_sm->server_buf_reader());
+ }
+
// Next order of business if copy the remaining data from the
// header buffer into new buffer
client_request_body_bytes += from_ua_buf->write(ua_txn->get_remote_reader());
diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h
index 8154d4d..9dd7375 100644
--- a/proxy/http/HttpSM.h
+++ b/proxy/http/HttpSM.h
@@ -61,6 +61,7 @@ static size_t const HTTP_SERVER_RESP_HDR_BUFFER_INDEX = BUFFER_SIZE_INDEX_8K;
class Http1ServerSession;
class AuthHttpAdapter;
+class PreWarmSM;
class HttpSM;
typedef int (HttpSM::*HttpSMHandler)(int event, void *data);
@@ -665,6 +666,7 @@ private:
int _client_transaction_priority_weight = -1, _client_transaction_priority_dependence = -1;
bool _from_early_data = false;
SNIRoutingType _tunnel_type = SNIRoutingType::NONE;
+ PreWarmSM *_prewarm_sm = nullptr;
};
////
diff --git a/proxy/http/Makefile.am b/proxy/http/Makefile.am
index a29d515..53f9832 100644
--- a/proxy/http/Makefile.am
+++ b/proxy/http/Makefile.am
@@ -77,13 +77,15 @@ libhttp_a_SOURCES = \
HttpTransactHeaders.h \
HttpTunnel.cc \
HttpTunnel.h \
- ForwardedConfig.cc
+ ForwardedConfig.cc \
+ PreWarmConfig.cc \
+ PreWarmManager.cc
if BUILD_TESTS
libhttp_a_SOURCES += RegressionHttpTransact.cc
endif
-check_PROGRAMS = test_proxy_http
+check_PROGRAMS = test_proxy_http test_PreWarm
TESTS = $(check_PROGRAMS)
@@ -111,6 +113,16 @@ test_proxy_http_LDADD = \
@HWLOC_LIBS@ \
@LIBCAP@
+test_PreWarm_CPPFLAGS = \
+ $(AM_CPPFLAGS) \
+ -I$(abs_top_srcdir)/tests/include
+
+test_PreWarm_LDADD = \
+ $(top_builddir)/src/tscore/libtscore.la
+
+test_PreWarm_SOURCES = \
+ unit_tests/test_PreWarm.cc
+
clang-tidy-local: $(libhttp_a_SOURCES) $(noinst_HEADERS)
$(CXX_Clang_Tidy)
diff --git a/proxy/http/PreWarmAlgorithm.h b/proxy/http/PreWarmAlgorithm.h
new file mode 100644
index 0000000..8b1d240
--- /dev/null
+++ b/proxy/http/PreWarmAlgorithm.h
@@ -0,0 +1,122 @@
+/** @file
+
+ Pre-Warming Pool Size Algorithm
+
+ v1: periodical pre-warming only
+ v2: periodical pre-warming + event based pre-warming
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#pragma once
+
+#include "tscore/ink_assert.h"
+#include "tscore/ink_error.h"
+
+#include <cstdint>
+#include <algorithm>
+
+namespace PreWarm
+{
+enum class Algorithm {
+ V1 = 1,
+ V2,
+};
+
+inline PreWarm::Algorithm
+algorithm_version(int i)
+{
+ switch (i) {
+ case 2:
+ return PreWarm::Algorithm::V2;
+ case 1:
+ return PreWarm::Algorithm::V1;
+ default:
+ ink_abort("unsupported version v=%d", i);
+ }
+}
+
+/**
+ Periodical pre-warming for algorithm v1
+
+ Expand the pool size to @requested_size
+
+ @params min : min connections (configured)
+ @params max : max connections (configured), -1 : unlimited
+
+ @return how many connections needs to be pre-warmed for next period
+ */
+inline uint32_t
+prewarm_size_v1_on_event_interval(uint32_t requested_size, uint32_t current_size, uint32_t min, int32_t max)
+{
+ uint32_t n = requested_size;
+
+ // keep tunnel_min connections pre-warmed at least
+ n = std::max(n, min);
+
+ if (max >= 0) {
+ n = std::min(n, static_cast<uint32_t>(max));
+ }
+
+ if (current_size >= n) {
+ // we already have enough connections, don't need to open new connection
+ return 0;
+ } else {
+ n -= current_size;
+ }
+
+ return n;
+}
+
+/**
+ Periodical pre-warming for algorithm v2
+
+ Expand the pool size to @current_size + @miss * @rate. The event based pre-warming handles the hit cases.
+
+ @params min : min connections (configured)
+ @params max : max connections (configured), -1 : unlimited
+
+ @return how many connections needs to be pre-warmed for next period
+ */
+inline uint32_t
+prewarm_size_v2_on_event_interval(uint32_t hit, uint32_t miss, uint32_t current_size, uint32_t min, int32_t max, double rate)
+{
+ if (hit + miss + current_size < min) {
+ // fallback to v1 to keep min size
+ return prewarm_size_v1_on_event_interval(hit + miss, current_size, min, max);
+ }
+
+ // Reached limit - do nothing
+ if (max >= 0 && current_size >= static_cast<uint32_t>(max)) {
+ return 0;
+ }
+
+ // Add #miss connections to the pool
+ uint32_t n = miss * rate;
+
+ // Check limit
+ if (max >= 0 && n + current_size > static_cast<uint32_t>(max)) {
+ ink_release_assert(static_cast<uint32_t>(max) > current_size);
+ n = max - current_size;
+ }
+
+ return n;
+}
+
+} // namespace PreWarm
diff --git a/proxy/http/PreWarmConfig.cc b/proxy/http/PreWarmConfig.cc
new file mode 100644
index 0000000..f214921
--- /dev/null
+++ b/proxy/http/PreWarmConfig.cc
@@ -0,0 +1,75 @@
+/** @file
+
+ Configs for PreWarming Tunnel
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "PreWarmConfig.h"
+#include "PreWarmManager.h"
+
+////
+// PreWarmConfigParams
+//
+PreWarmConfigParams::PreWarmConfigParams()
+{
+ // RECU_RESTART_TS
+ REC_EstablishStaticConfigByte(enabled, "proxy.config.tunnel.prewarm");
+ REC_EstablishStaticConfigInteger(max_stats_size, "proxy.config.tunnel.prewarm.max_stats_size");
+
+ // RECU_DYNAMIC
+ REC_ReadConfigInteger(event_period, "proxy.config.tunnel.prewarm.event_period");
+ REC_ReadConfigInteger(algorithm, "proxy.config.tunnel.prewarm.algorithm");
+}
+
+////
+// PreWarmConfig
+//
+void
+PreWarmConfig::startup()
+{
+ _config_update_handler = std::make_unique<ConfigUpdateHandler<PreWarmConfig>>();
+
+ // dynamic configs
+ _config_update_handler->attach("proxy.config.tunnel.prewarm.event_period");
+ _config_update_handler->attach("proxy.config.tunnel.prewarm.algorithm");
+
+ reconfigure();
+}
+
+void
+PreWarmConfig::reconfigure()
+{
+ PreWarmConfigParams *params = new PreWarmConfigParams();
+ _config_id = configProcessor.set(_config_id, params);
+
+ prewarmManager.reconfigure();
+}
+
+PreWarmConfigParams *
+PreWarmConfig::acquire()
+{
+ return static_cast<PreWarmConfigParams *>(configProcessor.get(_config_id));
+}
+
+void
+PreWarmConfig::release(PreWarmConfigParams *params)
+{
+ configProcessor.release(_config_id, params);
+}
diff --git a/proxy/http/PreWarmConfig.h b/proxy/http/PreWarmConfig.h
new file mode 100644
index 0000000..12b2210
--- /dev/null
+++ b/proxy/http/PreWarmConfig.h
@@ -0,0 +1,57 @@
+/** @file
+
+ Configs for PreWarming Tunnel
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "HttpConfig.h"
+
+struct PreWarmConfigParams : public ConfigInfo {
+ PreWarmConfigParams();
+
+ // noncopyable
+ PreWarmConfigParams(const HttpConfigParams &) = delete;
+ PreWarmConfigParams &operator=(const HttpConfigParams &) = delete;
+
+ // Config Params
+ int8_t enabled = 0;
+ int8_t algorithm = 0;
+ int64_t event_period = 0;
+ int64_t max_stats_size = 0;
+};
+
+class PreWarmConfig
+{
+public:
+ using scoped_config = ConfigProcessor::scoped_config<PreWarmConfig, PreWarmConfigParams>;
+
+ static void startup();
+
+ // ConfigUpdateContinuation interface
+ static void reconfigure();
+
+ // ConfigProcessor::scoped_config interface
+ static PreWarmConfigParams *acquire();
+ static void release(PreWarmConfigParams *params);
+
+private:
+ inline static int _config_id = 0;
+ inline static std::unique_ptr<ConfigUpdateHandler<PreWarmConfig>> _config_update_handler;
+};
diff --git a/proxy/http/PreWarmManager.cc b/proxy/http/PreWarmManager.cc
new file mode 100644
index 0000000..3a878d7
--- /dev/null
+++ b/proxy/http/PreWarmManager.cc
@@ -0,0 +1,1170 @@
+/** @file
+
+ Pre-Warming NetVConnection
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "PreWarmManager.h"
+#include "PreWarmConfig.h"
+
+#include "HttpConfig.h"
+#include "P_SSLSNI.h"
+
+#include "tscpp/util/PostScript.h"
+
+#include <algorithm>
+
+#define PreWarmSMDebug(fmt, ...) Debug("prewarm_sm", "[%p] " fmt, this, ##__VA_ARGS__);
+#define PreWarmSMVDebug(fmt, ...) Debug("v_prewarm_sm", "[%p] " fmt, this, ##__VA_ARGS__);
+
+ClassAllocator<PreWarmSM> preWarmSMAllocator("preWarmSMAllocator");
+PreWarmManager prewarmManager;
+
+namespace
+{
+using namespace std::literals;
+
+constexpr int DOWN_SERVER_TIMEOUT = 300;
+constexpr size_t STAT_NAME_BUF_LEN = 1024;
+
+constexpr std::string_view SRV_TUNNEL_TCP = "_tunnel._tcp."sv;
+constexpr std::string_view CLIENT_SNI_POLICY_SERVER_NAME = "server_name"sv;
+
+std::string_view
+alpn_name_for_stat(int alpn_id)
+{
+ if (alpn_id == TS_ALPN_PROTOCOL_INDEX_HTTP_1_0) {
+ return "http1_0"sv;
+ } else if (alpn_id == TS_ALPN_PROTOCOL_INDEX_HTTP_1_1) {
+ return "http1_1"sv;
+ } else if (alpn_id == TS_ALPN_PROTOCOL_INDEX_HTTP_2_0) {
+ return "http2"sv;
+ } else if (alpn_id == TS_ALPN_PROTOCOL_INDEX_HTTP_3) {
+ return "http3"sv;
+ } else {
+ return "unknown"sv;
+ }
+}
+
+void
+parse_authority(std::string &fqdn, int32_t &port, std::string_view authority)
+{
+ if (auto pos = authority.find(":"); pos != std::string::npos) {
+ fqdn = authority.substr(0, pos);
+ port = static_cast<in_port_t>(std::stoi(authority.substr(pos + 1).data()));
+ } else {
+ fqdn = authority;
+ port = -1;
+ }
+}
+
+////
+// Stats
+//
+constexpr std::string_view STAT_NAME_PREFIX = "proxy.process.tunnel.prewarm"sv;
+
+struct StatEntry {
+ std::string_view name;
+ RecRawStatSyncCb cb;
+};
+
+// the order is the same as PreWarm::Stat
+// clang-format off
+constexpr StatEntry STAT_ENTRIES[] = {
+ {"current_init"sv, RecRawStatSyncSum},
+ {"current_open"sv, RecRawStatSyncSum},
+ {"total_hit"sv, RecRawStatSyncSum},
+ {"total_miss"sv, RecRawStatSyncSum},
+ {"total_handshake_time"sv, RecRawStatSyncSum},
+ {"total_handshake_count"sv, RecRawStatSyncSum},
+ {"total_retry"sv, RecRawStatSyncSum},
+};
+// clang-format on
+
+} // namespace
+
+////
+// PreWarmSM
+//
+PreWarmSM::PreWarmSM(const PreWarm::SPtrConstDst &dst, const PreWarm::SPtrConstConf &conf,
+ const PreWarm::SPtrConstStatsIds &stats_ids)
+ : Continuation(new_ProxyMutex()), _dst(dst), _conf(conf), _stats_ids(stats_ids)
+{
+ SET_HANDLER(&PreWarmSM::state_init);
+
+ Debug("v_prewarm_conf", "host=%p _dst=%ld _conf=%ld _stats_ids=%ld", dst->host.data(), _dst.use_count(), _conf.use_count(),
+ _stats_ids.use_count());
+}
+
+PreWarmSM::~PreWarmSM() {}
+
+/**
+ Start opening netvc directly
+ */
+void
+PreWarmSM::start()
+{
+ _reset();
+ _retry_counter = 0;
+
+ handleEvent(EVENT_IMMEDIATE);
+}
+
+/**
+ Retry with Exponential Backoff (through EventProcessor)
+ */
+void
+PreWarmSM::retry()
+{
+ _reset();
+
+ ink_hrtime delay = HRTIME_SECONDS(1 << _retry_counter);
+ ++_retry_counter;
+ prewarmManager.stats.increment(_stats_ids->at(static_cast<int>(PreWarm::Stat::RETRY)), 1);
+
+ EThread *ethread = this_ethread();
+ _retry_event = ethread->schedule_in_local(this, delay, EVENT_IMMEDIATE);
+
+ if (_retry_counter % 10 == 0) {
+ Warning("retry pre-warming dst=%.*s:%d type=%d alpn=%d retry=%" PRIu32, (int)_dst->host.size(), _dst->host.data(), _dst->port,
+ (int)_dst->type, _dst->alpn_index, _retry_counter);
+ }
+}
+
+/**
+ Stop pre-warming. Move to state_closed from any state.
+ */
+void
+PreWarmSM::stop()
+{
+ if (handler == &PreWarmSM::state_closed) {
+ // do nothing
+ return;
+ }
+
+ _reset();
+ SET_HANDLER(&PreWarmSM::state_closed);
+ _milestones.mark(Milestone::CLOSED);
+}
+
+void
+PreWarmSM::destroy()
+{
+ _reset();
+
+ _dst.reset();
+ _conf.reset();
+ _stats_ids.reset();
+
+ this->mutex = nullptr;
+}
+
+/**
+ @brief Give ownership of netvc to the caller
+
+ PreWarmSM will not reveice any event as Continuation anymore.
+ Caller can read _read_buf through server_buf_reader().
+ */
+NetVConnection *
+PreWarmSM::move_netvc()
+{
+ if (handler != &PreWarmSM::state_open) {
+ return nullptr;
+ }
+
+ NetVConnection *netvc = _netvc;
+ _netvc = nullptr;
+
+ // clear the reference from netvc
+ netvc->do_io_read(nullptr, 0, nullptr);
+ netvc->do_io_write(nullptr, 0, nullptr);
+
+ _timeout.cancel_active_timeout();
+ _timeout.cancel_inactive_timeout();
+
+ if (_pending_action != nullptr) {
+ _pending_action->cancel();
+ _pending_action = nullptr;
+ }
+
+ SET_HANDLER(&PreWarmSM::state_closed);
+
+ return netvc;
+}
+
+int
+PreWarmSM::state_init(int event, void *data)
+{
+ switch (event) {
+ case EVENT_IMMEDIATE: {
+ if (_retry_event != nullptr && data == _retry_event) {
+ _retry_event = nullptr;
+ }
+
+ SET_HANDLER(&PreWarmSM::state_dns_lookup);
+ _timeout.set_active_timeout(_conf->connect_timeout);
+ _milestones.mark(Milestone::INIT);
+
+ PreWarmSMDebug("pre-warming a netvc dst=%.*s:%d type=%d alpn=%d retry=%" PRIu32, (int)_dst->host.size(), _dst->host.data(),
+ _dst->port, (int)_dst->type, _dst->alpn_index, _retry_counter);
+
+ if (_conf->srv_enabled) {
+ char target[MAXDNAME];
+ size_t target_len = 0;
+
+ memcpy(target, SRV_TUNNEL_TCP.data(), SRV_TUNNEL_TCP.size());
+ target_len += SRV_TUNNEL_TCP.size();
+
+ memcpy(target + target_len, _dst->host.data(), _dst->host.size());
+ target_len += _dst->host.size();
+
+ PreWarmSMVDebug("lookup SRV by %.*s", (int)target_len, target);
+
+ Action *srv_lookup_action_handle = hostDBProcessor.getSRVbyname_imm(
+ this, static_cast<cb_process_result_pfn>(&PreWarmSM::process_srv_info), target, target_len);
+ if (srv_lookup_action_handle != ACTION_RESULT_DONE) {
+ _pending_action = srv_lookup_action_handle;
+ }
+ } else {
+ PreWarmSMVDebug("lookup A/AAAA by %.*s", (int)_dst->host.size(), _dst->host.data());
+
+ Action *dns_lookup_action_handle = hostDBProcessor.getbyname_imm(
+ this, static_cast<cb_process_result_pfn>(&PreWarmSM::process_hostdb_info), _dst->host.data(), _dst->host.size());
+ if (dns_lookup_action_handle != ACTION_RESULT_DONE) {
+ _pending_action = dns_lookup_action_handle;
+ }
+ }
+
+ break;
+ }
+ default:
+ ink_abort("unsupported event=%s (%d)", get_vc_event_name(event), event);
+ break;
+ }
+
+ return EVENT_DONE;
+}
+
+int
+PreWarmSM::state_dns_lookup(int event, void *data)
+{
+ HostDBInfo *info = static_cast<HostDBInfo *>(data);
+
+ switch (event) {
+ case EVENT_HOST_DB_LOOKUP: {
+ _pending_action = nullptr;
+
+ if (info == nullptr || info->is_failed()) {
+ PreWarmSMVDebug("hostdb lookup is failed");
+
+ retry();
+ return EVENT_DONE;
+ }
+
+ IpEndpoint addr;
+
+ ats_ip_copy(addr, info->ip());
+ addr.port() = htons(_dst->port);
+
+ if (is_debug_tag_set("v_prewarm_sm")) {
+ char addrbuf[INET6_ADDRPORTSTRLEN];
+ PreWarmSMVDebug("hostdb lookup is done %s", ats_ip_nptop(addr, addrbuf, sizeof(addrbuf)));
+ }
+
+ SET_HANDLER(&PreWarmSM::state_net_open);
+ _milestones.mark(Milestone::DNS_LOOKUP_DONE);
+
+ Action *connect_action_handle = _connect(addr);
+ if (connect_action_handle != ACTION_RESULT_DONE) {
+ _pending_action = connect_action_handle;
+ }
+
+ break;
+ }
+ case EVENT_SRV_LOOKUP: {
+ _pending_action = nullptr;
+ std::string_view hostname;
+
+ if (info == nullptr || !info->is_srv || !info->round_robin) {
+ // no SRV record, fallback to default lookup
+ hostname = _dst->host;
+ } else {
+ HostDBRoundRobin *rr = info->rr();
+ HostDBInfo *srv = nullptr;
+ if (rr) {
+ char srv_hostname[MAXDNAME] = {0};
+
+ ink_hrtime now = Thread::get_hrtime();
+ srv = rr->select_best_srv(srv_hostname, &mutex->thread_holding->generator, ink_hrtime_to_sec(now), DOWN_SERVER_TIMEOUT);
+ hostname = std::string_view(srv_hostname);
+
+ if (srv == nullptr) {
+ // lookup SRV record failed, fallback to default lookup
+ hostname = _dst->host;
+ }
+ }
+ }
+
+ Action *dns_lookup_action_handle = hostDBProcessor.getbyname_imm(
+ this, static_cast<cb_process_result_pfn>(&PreWarmSM::process_hostdb_info), hostname.data(), hostname.size());
+ if (dns_lookup_action_handle != ACTION_RESULT_DONE) {
+ _pending_action = dns_lookup_action_handle;
+ }
+
+ break;
+ }
+ case VC_EVENT_ACTIVE_TIMEOUT:
+ retry();
+ break;
+ default:
+ ink_abort("unsupported event=%s (%d)", get_vc_event_name(event), event);
+ break;
+ }
+
+ return EVENT_DONE;
+}
+
+int
+PreWarmSM::state_net_open(int event, void *data)
+{
+ switch (event) {
+ case NET_EVENT_OPEN: {
+ _pending_action = nullptr;
+ _netvc = static_cast<NetVConnection *>(data);
+
+ // set buffers and (re)enable read/write for check status
+ // when TCP/TLS connection is established, VC_EVENT_WRITE_READY will be signaled
+ _read_buf = new_MIOBuffer(BUFFER_SIZE_INDEX_4K);
+ _read_buf_reader = _read_buf->alloc_reader();
+ _netvc->do_io_read(this, INT64_MAX, _read_buf);
+
+ _write_buf = new_MIOBuffer(BUFFER_SIZE_INDEX_4K);
+ _write_buf_reader = _write_buf->alloc_reader();
+ _netvc->do_io_write(this, INT64_MAX, _write_buf_reader);
+
+ break;
+ }
+ case VC_EVENT_READ_READY:
+ [[fallthrough]];
+ case VC_EVENT_WRITE_READY: {
+ VIO *vio = static_cast<VIO *>(data);
+ NetVConnection *netvc = static_cast<NetVConnection *>(vio->vc_server);
+
+ ink_release_assert(netvc == _netvc);
+ PreWarmSMVDebug("%s Handshake is done netvc=%p", (_dst->type == SNIRoutingType::FORWARD) ? "TCP" : "TLS", _netvc);
+
+ SET_HANDLER(&PreWarmSM::state_open);
+ _timeout.cancel_active_timeout();
+ _timeout.set_inactive_timeout(_conf->inactive_timeout);
+ _milestones.mark(Milestone::ESTABLISHED);
+ _record_handshake_time();
+
+ // disable write op of pre-warmed connection
+ // keep read op enabled to get EOS event from origin server
+ netvc->do_io_write(nullptr, 0, nullptr);
+
+ EThread *ethread = this_ethread();
+ ethread->prewarm_queue->push(_dst, this);
+
+ break;
+ }
+ case NET_EVENT_OPEN_FAILED: {
+ if (data != nullptr) {
+ const int errnum = -(reinterpret_cast<intptr_t>(data));
+ if (errnum == EADDRNOTAVAIL) {
+ // exhaust all ephemeral ports, do not retry
+ stop();
+ break;
+ }
+ Warning("NET_EVENT_OPEN_FAILED: error message=%s (%d)", strerror(errnum), errnum);
+ }
+ [[fallthrough]];
+ }
+ case VC_EVENT_ACTIVE_TIMEOUT:
+ [[fallthrough]];
+ case VC_EVENT_ERROR:
+ [[fallthrough]];
+ case VC_EVENT_EOS:
+ retry();
+ break;
+ default:
+ ink_abort("unsupported event=%s (%d)", get_vc_event_name(event), event);
+ break;
+ }
+
+ return EVENT_DONE;
+}
+
+int
+PreWarmSM::state_open(int event, void *data)
+{
+ switch (event) {
+ case VC_EVENT_READ_READY: {
+ // When the origin server sends something, keep it in the buffer. Forward it to the UA when a tunnel is setup
+ // (HttpSM::setup_blind_tunnel)
+ // - e.g. some HTTP/2 implementations send SETTINGS frame & WINDOW_UPDATE frame immediately when TLS handshake is done.
+ VIO *vio = static_cast<VIO *>(data);
+ NetVConnection *netvc = static_cast<NetVConnection *>(vio->vc_server);
+
+ ink_release_assert(netvc == _netvc);
+
+ if (is_debug_tag_set("prewarm_sm")) {
+ if (_read_buf_reader->is_read_avail_more_than(0)) {
+ uint64_t read_len = _read_buf_reader->read_avail();
+ PreWarmSMDebug("buffering data from origin server len=%" PRIu64, read_len);
+
+ if (is_debug_tag_set("v_prewarm_sm")) {
+ uint8_t buf[1024];
+ read_len = std::min(static_cast<uint64_t>(sizeof(buf)), read_len);
+ _read_buf_reader->memcpy(buf, read_len);
+
+ ts::LocalBufferWriter<2048> bw;
+ bw.print("{}", ts::bwf::Hex_Dump(buf));
+
+ PreWarmSMVDebug("\n%.*s\n", (int)read_len * 2, bw.data());
+ }
+ }
+ }
+
+ break;
+ }
+ case VC_EVENT_EOS:
+ // possibly inactive timeout at origin server
+ [[fallthrough]];
+ case VC_EVENT_INACTIVITY_TIMEOUT: {
+ PreWarmSMDebug("%s (%d)", get_vc_event_name(event), event);
+ stop();
+ break;
+ }
+ case VC_EVENT_ACTIVE_TIMEOUT:
+ [[fallthrough]];
+ case VC_EVENT_ERROR:
+ [[fallthrough]];
+ case VC_EVENT_WRITE_READY:
+ [[fallthrough]];
+ default:
+ ink_abort("unsupported event=%s (%d)", get_vc_event_name(event), event);
+ break;
+ }
+
+ return EVENT_DONE;
+}
+
+int
+PreWarmSM::state_closed(int event, void *data)
+{
+ switch (event) {
+ default:
+ ink_abort("unsupported event=%s (%d)", get_vc_event_name(event), event);
+ break;
+ }
+
+ return EVENT_DONE;
+}
+
+IOBufferReader *
+PreWarmSM::server_buf_reader()
+{
+ return _read_buf_reader;
+}
+
+bool
+PreWarmSM::has_data_from_origin_server() const
+{
+ if (_read_buf_reader == nullptr) {
+ return false;
+ }
+
+ return _read_buf_reader->is_read_avail_more_than(0);
+}
+
+bool
+PreWarmSM::is_active_timeout_expired(ink_hrtime now)
+{
+ return _timeout.is_active_timeout_expired(now);
+}
+
+bool
+PreWarmSM::is_inactive_timeout_expired(ink_hrtime now)
+{
+ return _timeout.is_inactive_timeout_expired(now);
+}
+
+void
+PreWarmSM::process_hostdb_info(HostDBInfo *r)
+{
+ ink_release_assert(this->handler == &PreWarmSM::state_dns_lookup);
+
+ this->handleEvent(EVENT_HOST_DB_LOOKUP, r);
+}
+
+void
+PreWarmSM::process_srv_info(HostDBInfo *r)
+{
+ ink_release_assert(this->handler == &PreWarmSM::state_dns_lookup);
+
+ this->handleEvent(EVENT_SRV_LOOKUP, r);
+}
+
+Action *
+PreWarmSM::_connect(const IpEndpoint &addr)
+{
+ Action *connect_action_handle = nullptr;
+
+ HttpConfig::scoped_config http_conf_params;
+
+ NetVCOptions opt;
+ opt.reset();
+ opt.f_blocking_connect = false;
+ opt.set_sock_param(http_conf_params->oride.sock_recv_buffer_size_out, http_conf_params->oride.sock_send_buffer_size_out,
+ http_conf_params->oride.sock_option_flag_out, http_conf_params->oride.sock_packet_mark_out,
+ http_conf_params->oride.sock_packet_tos_out);
+ opt.f_tcp_fastopen = (http_conf_params->oride.sock_option_flag_out & NetVCOptions::SOCK_OPT_TCP_FAST_OPEN);
+
+ switch (_dst->type) {
+ case SNIRoutingType::FORWARD: {
+ SCOPED_MUTEX_LOCK(lock, mutex, this_ethread());
+ // TODO: constify UnixNetProcessor::connect_re_internal()
+ connect_action_handle = netProcessor.connect_re(this, &addr.sa, &opt);
+ break;
+ }
+ case SNIRoutingType::PARTIAL_BLIND: {
+ // SNI
+ opt.set_sni_servername(_conf->sni.data(), _conf->sni.size());
+
+ // ALPN
+ opt.alpn_protos = SessionProtocolNameRegistry::convert_openssl_alpn_wire_format(_dst->alpn_index);
+
+ // Verify Server Configs
+ opt.verifyServerPolicy = _conf->verify_server_policy;
+ opt.verifyServerProperties = _conf->verify_server_properties;
+
+ // Client Cert
+ opt.ssl_client_cert_name = http_conf_params->oride.ssl_client_cert_filename;
+ opt.ssl_client_private_key_name = http_conf_params->oride.ssl_client_private_key_filename;
+ opt.ssl_client_ca_cert_name = http_conf_params->oride.ssl_client_ca_cert_filename;
+
+ SCOPED_MUTEX_LOCK(lock, mutex, this_ethread());
+ connect_action_handle = sslNetProcessor.connect_re(this, &addr.sa, &opt);
+ break;
+ }
+ default:
+ // do nothing
+ break;
+ }
+
+ return connect_action_handle;
+}
+
+/**
+ Reset state & *some* members
+ */
+void
+PreWarmSM::_reset()
+{
+ SET_HANDLER(&PreWarmSM::state_init);
+
+ _timeout.cancel_active_timeout();
+ _timeout.cancel_inactive_timeout();
+
+ if (_netvc != nullptr) {
+ _netvc->do_io_close();
+ _netvc = nullptr;
+ }
+
+ if (_pending_action != nullptr) {
+ _pending_action->cancel();
+ _pending_action = nullptr;
+ }
+
+ if (_read_buf != nullptr) {
+ // free_MIOBuffer dealloc all readers
+ free_MIOBuffer(_read_buf);
+ _read_buf = nullptr;
+ _read_buf_reader = nullptr;
+ }
+
+ if (_write_buf != nullptr) {
+ // free_MIOBuffer dealloc all readers
+ free_MIOBuffer(_write_buf);
+ _write_buf = nullptr;
+ _write_buf_reader = nullptr;
+ }
+
+ if (_retry_event != nullptr) {
+ _retry_event->cancel();
+ _retry_event = nullptr;
+ }
+}
+
+void
+PreWarmSM::_record_handshake_time()
+{
+ ink_hrtime duration = _milestones.elapsed(Milestone::INIT, Milestone::ESTABLISHED);
+
+ ink_assert(duration > 0);
+ if (duration <= 0) {
+ return;
+ }
+
+ prewarmManager.stats.increment(_stats_ids->at(static_cast<int>(PreWarm::Stat::HANDSHAKE_TIME)), duration);
+ prewarmManager.stats.increment(_stats_ids->at(static_cast<int>(PreWarm::Stat::HANDSHAKE_COUNT)), 1);
+}
+
+////
+// PreWarmQueue
+//
+PreWarmQueue::PreWarmQueue() : Continuation(new_ProxyMutex())
+{
+ SET_HANDLER(&PreWarmQueue::state_init);
+}
+
+PreWarmQueue::~PreWarmQueue()
+{
+ _tick_event->cancel();
+ _tick_event = nullptr;
+
+ for (auto &e : _map) {
+ Info info = e.second;
+
+ _make_queue_empty(info.init_list);
+ delete info.init_list;
+
+ _make_queue_empty(info.open_list);
+ delete info.open_list;
+ }
+
+ this->mutex = nullptr;
+}
+
+int
+PreWarmQueue::state_init(int event, void *data)
+{
+ switch (event) {
+ case EVENT_IMMEDIATE: {
+ _reconfigure();
+
+ _cop = ActivityCop<PreWarmSM>(this->mutex, &_cop_list, 1);
+ _cop.start();
+
+ // schedule tick event
+ EThread *ethread = this_ethread();
+ _tick_event = ethread->schedule_every_local(this, _event_period);
+
+ SET_HANDLER(&PreWarmQueue::state_running);
+
+ break;
+ }
+ default:
+ ink_abort("unsupported event=%s (%d)", get_vc_event_name(event), event);
+ break;
+ }
+
+ return EVENT_DONE;
+}
+
+int
+PreWarmQueue::state_running(int event, void *data)
+{
+ switch (event) {
+ case EVENT_INTERVAL: {
+ for (auto &[dst, info] : _map) {
+ // mentain queues
+ _delete_closed_sm(info.init_list);
+ _delete_closed_sm(info.open_list);
+
+ // pre-warm new connections
+ _prewarm_on_event_interval(dst, info);
+
+ // set prewarmManager.stats
+ Debug("v_prewarm_q", "dst=%.*s:%d type=%d alpn=%d miss=%d hit=%d init=%d open=%d", (int)dst->host.size(), dst->host.data(),
+ dst->port, (int)dst->type, dst->alpn_index, info.stat.miss, info.stat.hit, (int)info.init_list->size(),
+ (int)info.open_list->size());
+
+ prewarmManager.stats.set_sum(info.stats_ids->at(static_cast<int>(PreWarm::Stat::INIT_LIST_SIZE)), info.init_list->size());
+ prewarmManager.stats.set_sum(info.stats_ids->at(static_cast<int>(PreWarm::Stat::OPEN_LIST_SIZE)), info.open_list->size());
+ prewarmManager.stats.increment(info.stats_ids->at(static_cast<int>(PreWarm::Stat::HIT)), info.stat.hit);
+ prewarmManager.stats.increment(info.stats_ids->at(static_cast<int>(PreWarm::Stat::MISS)), info.stat.miss);
+
+ // clear PreWarmQueue::Stat
+ info.stat.miss = 0;
+ info.stat.hit = 0;
+ }
+ break;
+ }
+ case EVENT_IMMEDIATE: {
+ _reconfigure();
+
+ // reschedule tick event
+ EThread *ethread = this_ethread();
+ _tick_event->cancel();
+ _tick_event = ethread->schedule_every_local(this, _event_period);
+
+ break;
+ }
+ default:
+ ink_abort("unsupported event=%s (%d)", get_vc_event_name(event), event);
+ break;
+ }
+
+ return EVENT_DONE;
+}
+
+void
+PreWarmQueue::push(const PreWarm::SPtrConstDst &dst, PreWarmSM *sm)
+{
+ ink_release_assert(sm->handler == &PreWarmSM::state_open);
+
+ if (auto res = _map.find(dst); res != _map.end()) {
+ Queue *init_list = res->second.init_list;
+
+ // expecting init_list.front() is sm in many cases, if not we need to change container
+ for (auto it = init_list->begin(); it != init_list->end(); ++it) {
+ if (*it == sm) {
+ it = init_list->erase(it);
+ break;
+ }
+ }
+
+ res->second.open_list->push_front(sm);
+ }
+}
+
+/**
+ Use open_list as FILO to adjust size of list. ( A new sm is pushed in front of the list by PreWarmQeueu::push() )
+ When the list has redundant sm(s), they will be closed by inactivity timeout.
+ */
+PreWarmSM *
+PreWarmQueue::dequeue(const PreWarm::SPtrConstDst &target)
+{
+ PreWarmSM *sm = nullptr;
+
+ auto res = _map.find(target);
+ if (res == _map.end()) {
+ // no such pool
+ return nullptr;
+ }
+
+ const PreWarm::SPtrConstDst &dst = res->first;
+ Info &info = res->second;
+
+ Queue *q = info.open_list;
+ while (!q->empty()) {
+ sm = q->front();
+ q->pop_front();
+
+ if (sm->handler == &PreWarmSM::state_open) {
+ _cop_list.remove(sm);
+ break;
+ }
+
+ _delete_prewarm_sm(sm);
+ sm = nullptr;
+ }
+
+ // stat
+ if (sm == nullptr) {
+ ++info.stat.miss;
+ } else {
+ ++info.stat.hit;
+ }
+
+ _prewarm_on_dequeue(dst, info);
+
+ return sm;
+}
+
+void
+PreWarmQueue::_new_prewarm_sm(const PreWarm::SPtrConstDst &dst, const PreWarm::SPtrConstConf &conf,
+ const PreWarm::SPtrConstStatsIds &stats_ids)
+{
+ EThread *ethread = this_ethread();
+
+ PreWarmSM *sm = THREAD_ALLOC(preWarmSMAllocator, ethread);
+ new (sm) PreWarmSM(dst, conf, stats_ids);
+ _cop_list.push(sm);
+
+ _map[dst].init_list->push_back(sm);
+
+ SCOPED_MUTEX_LOCK(lock, sm->mutex, ethread);
+ sm->start();
+}
+
+void
+PreWarmQueue::_delete_prewarm_sm(PreWarmSM *sm)
+{
+ ink_release_assert(sm->handler == &PreWarmSM::state_closed);
+
+ _cop_list.remove(sm);
+
+ sm->destroy();
+ THREAD_FREE(sm, preWarmSMAllocator, this_ethread());
+}
+
+/**
+ Periodical pre-warming
+
+ Try to keep (min <= pool size && pool size <= max)
+
+ V1: Expand the pool size to requested size
+ V2: Expand the pool size to current size + miss * rate
+ */
+void
+PreWarmQueue::_prewarm_on_event_interval(const PreWarm::SPtrConstDst &dst, const Info &info)
+{
+ const uint32_t current_size = info.init_list->size() + info.open_list->size();
+ uint32_t n = 0;
+
+ switch (_algorithm) {
+ case PreWarm::Algorithm::V2: {
+ n = PreWarm::prewarm_size_v2_on_event_interval(info.stat.hit, info.stat.miss, current_size, info.conf->min, info.conf->max,
+ info.conf->rate);
+ break;
+ }
+ case PreWarm::Algorithm::V1:
+ [[fallthrough]];
+ default:
+ n = PreWarm::prewarm_size_v1_on_event_interval(info.stat.miss + info.stat.hit, current_size, info.conf->min, info.conf->max);
+ break;
+ }
+
+ Debug("v_prewarm_q", "prewarm_size=%" PRId32, n);
+
+ for (uint32_t i = 0; i < n; ++i) {
+ _new_prewarm_sm(dst, info.conf, info.stats_ids);
+ }
+}
+
+/**
+ Event based pre-warming
+
+ V1: Do nothing
+ V2: Start pre-warming a new netvc
+ */
+void
+PreWarmQueue::_prewarm_on_dequeue(const PreWarm::SPtrConstDst &dst, const Info &info)
+{
+ switch (_algorithm) {
+ case PreWarm::Algorithm::V2: {
+ const int32_t current_size = info.init_list->size() + info.open_list->size();
+ if (current_size < info.conf->max) {
+ _new_prewarm_sm(dst, info.conf, info.stats_ids);
+ }
+ break;
+ }
+ case PreWarm::Algorithm::V1:
+ [[fallthrough]];
+ default:
+ // do nothing
+ break;
+ }
+}
+
+/**
+ Reconfigure _map based on new SNIConfig
+ */
+void
+PreWarmQueue::_reconfigure()
+{
+ {
+ PreWarmConfig::scoped_config prewarm_conf;
+
+ _event_period = HRTIME_MSECONDS(prewarm_conf->event_period);
+ _algorithm = PreWarm::algorithm_version(prewarm_conf->algorithm);
+ }
+
+ // build new map based on new SNIConfig
+ const PreWarm::ParsedSNIConf &new_conf_list = prewarmManager.get_parsed_conf();
+ const PreWarm::StatsIdMap &new_stats_id_map = prewarmManager.get_stats_id_map();
+
+ Map new_map;
+
+ for (auto &entry : new_conf_list) {
+ const PreWarm::SPtrConstDst &dst = entry.first;
+ PreWarm::SPtrConstConf conf = entry.second;
+
+ if (const auto &res = _map.find(dst); res != _map.end()) {
+ // copy from old info
+ const Info &old_info = res->second;
+
+ new_map[dst] = Info{old_info.init_list, old_info.open_list, conf, old_info.stats_ids, old_info.stat};
+ } else {
+ // make new info
+ PreWarm::SPtrConstStatsIds stats_ids;
+ if (const auto &res = new_stats_id_map.find(dst); res != new_stats_id_map.end()) {
+ stats_ids = res->second;
+ } else {
+ Error("no stats ids found for %s", dst->host.c_str());
+ continue;
+ }
+
+ Queue *init_list = new Queue();
+ Queue *open_list = new Queue();
+ new_map[dst] = Info{init_list, open_list, conf, stats_ids, {}};
+ }
+ }
+
+ // free unexisting entries
+ for (auto &[dst, info] : _map) {
+ if (auto entry = new_conf_list.find(dst); entry == new_conf_list.end()) {
+ prewarmManager.stats.set_sum(info.stats_ids->at(static_cast<int>(PreWarm::Stat::INIT_LIST_SIZE)), 0);
+ prewarmManager.stats.set_sum(info.stats_ids->at(static_cast<int>(PreWarm::Stat::OPEN_LIST_SIZE)), 0);
+
+ _make_queue_empty(info.init_list);
+ delete info.init_list;
+
+ _make_queue_empty(info.open_list);
+ delete info.open_list;
+
+ info.conf.reset();
+ info.stats_ids.reset();
+ }
+ }
+
+ std::swap(_map, new_map);
+}
+
+/**
+ Delete all PreWarmSM in the queue
+ */
+void
+PreWarmQueue::_make_queue_empty(Queue *q)
+{
+ while (!q->empty()) {
+ PreWarmSM *sm = q->front();
+ q->pop_front();
+ sm->stop();
+ _delete_prewarm_sm(sm);
+ }
+}
+
+/**
+ Delete closed state PreWarmSM in the queue
+ */
+void
+PreWarmQueue::_delete_closed_sm(Queue *q)
+{
+ for (auto it = q->begin(); it != q->end();) {
+ if ((*it)->handler == &PreWarmSM::state_closed) {
+ _delete_prewarm_sm(*it);
+ it = q->erase(it);
+ } else {
+ ++it;
+ }
+ }
+}
+
+////
+// PreWarmManager
+//
+void
+PreWarmManager::reconfigure_prewarming_on_threads()
+{
+ EventProcessor::ThreadGroupDescriptor *tg = &(eventProcessor.thread_group[0]);
+ ink_release_assert(memcmp(tg->_name.data(), "ET_NET", 6) == 0);
+
+ Debug("prewarm", "reconfigure prewarming");
+
+ for (int i = 0; i < tg->_count; ++i) {
+ EThread *ethread = tg->_thread[i];
+ if (ethread->prewarm_queue == nullptr) {
+ ethread->prewarm_queue = new PreWarmQueue();
+ }
+ ethread->schedule_imm_local(ethread->prewarm_queue);
+ }
+}
+
+void
+PreWarmManager::start()
+{
+ PreWarmConfig::startup();
+
+ _mutex = new_ProxyMutex();
+
+ this->reconfigure();
+}
+
+void
+PreWarmManager::reconfigure()
+{
+ if (_mutex == nullptr) {
+ // don't reconfigure before start
+ return;
+ }
+
+ SCOPED_MUTEX_LOCK(lock, _mutex, this_ethread());
+
+ PreWarmConfig::scoped_config prewarm_conf;
+ bool is_prewarm_enabled = prewarm_conf->enabled;
+
+ SNIConfig::scoped_config sni_conf;
+ for (const auto &item : sni_conf->Y_sni.items) {
+ if (item.tunnel_prewarm == YamlSNIConfig::TunnelPreWarm::ENABLED) {
+ is_prewarm_enabled = true;
+ break;
+ }
+ }
+
+ if (is_prewarm_enabled) {
+ if (!stats.is_allocated()) {
+ stats.init(prewarm_conf->max_stats_size);
+ }
+
+ _parsed_conf.clear();
+ _parse_sni_conf(_parsed_conf, sni_conf);
+ _register_stats(_parsed_conf);
+
+ reconfigure_prewarming_on_threads();
+ }
+}
+
+/**
+ TODO: stop pre-warming
+ */
+void
+PreWarmManager::stop()
+{
+ _mutex->free();
+}
+
+const PreWarm::ParsedSNIConf &
+PreWarmManager::get_parsed_conf() const
+{
+ return _parsed_conf;
+}
+
+const PreWarm::StatsIdMap &
+PreWarmManager::get_stats_id_map() const
+{
+ return _stats_id_map;
+}
+
+/**
+ Convert SNIConfigParams to PreWarm::ParsedSNIConf
+ */
+void
+PreWarmManager::_parse_sni_conf(PreWarm::ParsedSNIConf &parsed_conf, const SNIConfigParams *sni_conf) const
+{
+ PreWarmConfig::scoped_config prewarm_conf;
+
+ for (const auto &item : sni_conf->Y_sni.items) {
+ if (item.tunnel_type != SNIRoutingType::FORWARD && item.tunnel_type != SNIRoutingType::PARTIAL_BLIND) {
+ continue;
+ }
+
+ if (item.tunnel_prewarm == YamlSNIConfig::TunnelPreWarm::DISABLED ||
+ (item.tunnel_prewarm == YamlSNIConfig::TunnelPreWarm::UNSET && !prewarm_conf->enabled)) {
+ continue;
+ }
+
+ std::vector<int> alpn_ids = {SessionProtocolNameRegistry::INVALID};
+ if (!item.tunnel_alpn.empty() && item.tunnel_type == SNIRoutingType::PARTIAL_BLIND) {
+ for (auto &id : item.tunnel_alpn) {
+ alpn_ids.push_back(id);
+ }
+ }
+
+ for (int id : alpn_ids) {
+ Debug("prewarm_m", "sni=%s dst=%s type=%d alpn=%d min=%d max=%d c_timeout=%d i_timeout=%d srv=%d", item.fqdn.c_str(),
+ item.tunnel_destination.c_str(), (int)item.tunnel_type, id, (int)item.tunnel_prewarm_min, (int)item.tunnel_prewarm_max,
+ (int)item.tunnel_prewarm_connect_timeout, (int)item.tunnel_prewarm_inactive_timeout, item.tunnel_prewarm_srv);
+
+ std::string dst_fqdn;
+ int32_t port;
+ parse_authority(dst_fqdn, port, item.tunnel_destination);
+
+ if (port < 0) {
+ if (item.tunnel_type == SNIRoutingType::PARTIAL_BLIND) {
+ port = 443;
+ } else {
+ port = 80;
+ }
+ }
+
+ PreWarm::SPtrConstDst dst = std::make_shared<const PreWarm::Dst>(dst_fqdn, port, item.tunnel_type, id);
+
+ // clang-format off
+ PreWarm::SPtrConstConf conf = std::make_shared<const PreWarm::Conf>(
+ item.tunnel_prewarm_min,
+ item.tunnel_prewarm_max,
+ item.tunnel_prewarm_rate,
+ HRTIME_SECONDS(item.tunnel_prewarm_connect_timeout),
+ HRTIME_SECONDS(item.tunnel_prewarm_inactive_timeout),
+ item.tunnel_prewarm_srv,
+ item.verify_server_policy,
+ item.verify_server_properties,
+ (strcmp(item.client_sni_policy, CLIENT_SNI_POLICY_SERVER_NAME) == 0)? item.fqdn : dst_fqdn
+ );
+ // clang-format on
+
+ parsed_conf[dst] = conf;
+ }
+ }
+}
+
+/**
+ Create stats per pool.
+ Registered stats id is stored in _stat_id_map.
+ */
+void
+PreWarmManager::_register_stats(const PreWarm::ParsedSNIConf &parsed_conf)
+{
+ int stats_counter = 0;
+
+ for (auto &entry : parsed_conf) {
+ const PreWarm::SPtrConstDst &dst = entry.first;
+
+ PreWarm::StatsIds ids;
+ for (int j = 0; j < static_cast<int>(PreWarm::Stat::LAST_ENTRY); ++j) {
+ char name[STAT_NAME_BUF_LEN];
+
+ if (dst->alpn_index != SessionProtocolNameRegistry::INVALID) {
+ std::string_view alpn_name = alpn_name_for_stat(dst->alpn_index);
+
+ snprintf(name, sizeof(name), "%s.%.*s:%d.tls.%s.%s", STAT_NAME_PREFIX.data(), static_cast<int>(dst->host.size()),
+ dst->host.data(), dst->port, alpn_name.data(), STAT_ENTRIES[j].name.data());
+ } else {
+ snprintf(name, sizeof(name), "%s.%.*s:%d.%s.%s", STAT_NAME_PREFIX.data(), static_cast<int>(dst->host.size()),
+ dst->host.data(), dst->port, (dst->type == SNIRoutingType::PARTIAL_BLIND) ? "tls" : "tcp",
+ STAT_ENTRIES[j].name.data());
+ }
+
+ int stats_id = stats.find(name);
+ if (stats_id < 0) {
+ stats_id = stats.create(RECT_PROCESS, name, RECD_INT, STAT_ENTRIES[j].cb);
+
+ if (stats_id < 0) {
+ // proxy.config.tunnel.prewarm.max_stats_size is enough?
+ Error("couldn't register stat name=%s", name);
+ } else {
+ ++stats_counter;
+ }
+ }
+
+ ids[j] = stats_id;
+
+ Debug("v_prewarm_init", "stat id=%d name=%s", stats_id, name);
+ }
+
+ _stats_id_map[dst] = std::make_shared<const PreWarm::StatsIds>(ids);
+ }
+
+ Note("%d dynamic stats are registered for pre-warming tunnel", stats_counter);
+}
diff --git a/proxy/http/PreWarmManager.h b/proxy/http/PreWarmManager.h
new file mode 100644
index 0000000..e7da819
--- /dev/null
+++ b/proxy/http/PreWarmManager.h
@@ -0,0 +1,341 @@
+/** @file
+
+ Pre-Warming NetVConnection
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#pragma once
+
+#include "PreWarmAlgorithm.h"
+
+#include "P_Net.h"
+#include "I_NetVConnection.h"
+#include "P_SSLSNI.h"
+#include "P_HostDB.h"
+#include "YamlSNIConfig.h"
+#include "NetTimeout.h"
+#include "Milestones.h"
+
+#include "records/DynamicStats.h"
+
+#include <map>
+#include <memory>
+#include <queue>
+#include <string_view>
+
+class PreWarmSM;
+class PreWarmManager;
+
+extern ClassAllocator<PreWarmSM> preWarmSMAllocator;
+extern PreWarmManager prewarmManager;
+
+namespace PreWarm
+{
+////
+// Dst
+//
+struct Dst {
+ Dst(const std::string &h, in_port_t p, SNIRoutingType t, int a) : host(h), port(p), type(t), alpn_index(a) {}
+
+ std::string host;
+ in_port_t port = 0;
+ SNIRoutingType type = SNIRoutingType::NONE;
+ int alpn_index = SessionProtocolNameRegistry::INVALID;
+};
+
+using SPtrConstDst = std::shared_ptr<const Dst>;
+
+struct DstHash {
+ size_t
+ operator()(const PreWarm::SPtrConstDst &dst) const
+ {
+ CryptoHash hash;
+ CryptoContext context{};
+
+ context.update(dst->host.data(), dst->host.size());
+ context.update(&dst->port, sizeof(in_port_t));
+ context.update(&dst->type, sizeof(SNIRoutingType));
+ context.update(&dst->alpn_index, sizeof(int));
+
+ context.finalize(hash);
+
+ return static_cast<size_t>(hash.fold());
+ }
+};
+
+struct DstKeyEqual {
+ bool
+ operator()(const PreWarm::SPtrConstDst &x, const PreWarm::SPtrConstDst &y) const
+ {
+ return x->host == y->host && x->port == y->port && x->type == y->type && x->alpn_index == y->alpn_index;
+ }
+};
+
+////
+// Conf
+//
+struct Conf {
+ Conf(uint32_t min, int32_t max, double rate, ink_hrtime connect_timeout, ink_hrtime inactive_timeout, bool srv_enabled,
+ YamlSNIConfig::Policy verify_server_policy, YamlSNIConfig::Property verify_server_properties, const std::string &sni)
+ : min(min),
+ max(max),
+ rate(rate),
+ connect_timeout(connect_timeout),
+ inactive_timeout(inactive_timeout),
+ srv_enabled(srv_enabled),
+ verify_server_policy(verify_server_policy),
+ verify_server_properties(verify_server_properties),
+ sni(sni)
+ {
+ }
+
+ uint32_t min = 0;
+ int32_t max = 0;
+ double rate = 1.0;
+ ink_hrtime connect_timeout = 0;
+ ink_hrtime inactive_timeout = 0;
+ bool srv_enabled = false;
+ YamlSNIConfig::Policy verify_server_policy = YamlSNIConfig::Policy::UNSET;
+ YamlSNIConfig::Property verify_server_properties = YamlSNIConfig::Property::UNSET;
+ std::string sni;
+};
+
+using SPtrConstConf = std::shared_ptr<const Conf>;
+using ParsedSNIConf = std::unordered_map<SPtrConstDst, SPtrConstConf, DstHash, DstKeyEqual>;
+
+////
+// Stats
+//
+enum class Stat {
+ INIT_LIST_SIZE = 0,
+ OPEN_LIST_SIZE,
+ HIT,
+ MISS,
+ HANDSHAKE_TIME,
+ HANDSHAKE_COUNT,
+ RETRY,
+ LAST_ENTRY,
+};
+
+using StatsIds = std::array<int, static_cast<size_t>(PreWarm::Stat::LAST_ENTRY)>;
+using SPtrConstStatsIds = std::shared_ptr<const StatsIds>;
+using StatsIdMap = std::unordered_map<SPtrConstDst, SPtrConstStatsIds, DstHash, DstKeyEqual>;
+
+} // namespace PreWarm
+
+/**
+ @class PreWarmSM
+ @brief A state machine to pre-warm connection
+
+ @startuml
+ hide empty description
+ [*] --> state_init : new
+ state_init --> state_dns_lookup : start()
+ state_init --> state_closed : stop()
+ state_dns_lookup --> state_net_open : HostDB lookup is done
+ state_dns_lookup --> state_init : retry()
+ state_dns_lookup --> state_closed : stop()
+ state_net_open --> state_open : TCP/TLS Handshake is done
+ state_net_open --> state_init : retry()
+ state_net_open --> state_closed : stop()
+ state_open --> state_closed : move_netvc()\nstop()
+ state_closed --> [*] : delete
+ @enduml
+ */
+class PreWarmSM : public Continuation
+{
+public:
+ PreWarmSM(){};
+ PreWarmSM(const PreWarm::SPtrConstDst &dst, const PreWarm::SPtrConstConf &conf, const PreWarm::SPtrConstStatsIds &stats_ids);
+ ~PreWarmSM() override;
+
+ // States
+ int state_init(int event, void *data);
+ int state_dns_lookup(int event, void *data);
+ int state_net_open(int event, void *data);
+ int state_open(int event, void *data);
+ int state_closed(int event, void *data);
+
+ // Controllers
+ void start();
+ void retry();
+ void stop();
+ void destroy();
+
+ // Modifiers
+ NetVConnection *move_netvc();
+ IOBufferReader *server_buf_reader();
+
+ // References
+ bool has_data_from_origin_server() const;
+
+ // NetTimeout
+ // TODO: constify
+ bool is_active_timeout_expired(ink_hrtime now);
+ bool is_inactive_timeout_expired(ink_hrtime now);
+
+ // HostDB inline completion functions
+ void process_hostdb_info(HostDBInfo *r);
+ void process_srv_info(HostDBInfo *r);
+
+private:
+ enum class Milestone {
+ INIT = 0,
+ DNS_LOOKUP_DONE,
+ ESTABLISHED,
+ CLOSED,
+ LAST_ENTRY,
+ };
+
+ Action *_connect(const IpEndpoint &addr);
+ void _reset();
+ void _record_handshake_time();
+
+ ////
+ // Variables
+ //
+ NetTimeout _timeout{};
+ Milestones<Milestone, static_cast<size_t>(Milestone::LAST_ENTRY)> _milestones;
+
+ uint32_t _retry_counter = 0;
+
+ PreWarm::SPtrConstDst _dst;
+ PreWarm::SPtrConstConf _conf;
+ PreWarm::SPtrConstStatsIds _stats_ids;
+
+ NetVConnection *_netvc = nullptr;
+ Action *_pending_action = nullptr;
+ MIOBuffer *_read_buf = nullptr;
+ IOBufferReader *_read_buf_reader = nullptr;
+ MIOBuffer *_write_buf = nullptr;
+ IOBufferReader *_write_buf_reader = nullptr;
+ Event *_retry_event = nullptr;
+};
+
+/**
+ @class PreWarmQueue
+ @detail
+ - Each ET_NET thread has this queue
+ - Responsible for the life cycle of PreWarmSM until giving it to HttpSM
+
+ @startuml
+ hide empty description
+ [*] --> state_init : new
+ state_init --> state_running : start pre-warming
+ @enduml
+ */
+class PreWarmQueue : public Continuation
+{
+public:
+ PreWarmQueue();
+ ~PreWarmQueue();
+
+ // States
+ int state_init(int event, void *data);
+ int state_running(int event, void *data);
+
+ // Modifiers for queue
+ void push(const PreWarm::SPtrConstDst &dst, PreWarmSM *sm);
+ PreWarmSM *dequeue(const PreWarm::SPtrConstDst &dst);
+
+private:
+ using Queue = std::deque<PreWarmSM *>;
+
+ struct Stat {
+ uint32_t miss = 0;
+ uint32_t hit = 0;
+ };
+
+ struct Info {
+ Queue *init_list;
+ Queue *open_list;
+ PreWarm::SPtrConstConf conf;
+ PreWarm::SPtrConstStatsIds stats_ids;
+ Stat stat;
+ };
+
+ using Map = std::unordered_map<PreWarm::SPtrConstDst, Info, PreWarm::DstHash, PreWarm::DstKeyEqual>;
+
+ // construct/destruct PreWarmSM
+ void _new_prewarm_sm(const PreWarm::SPtrConstDst &dst, const PreWarm::SPtrConstConf &conf,
+ const PreWarm::SPtrConstStatsIds &stats_ids);
+ void _delete_prewarm_sm(PreWarmSM *sm);
+
+ void _reconfigure();
+ void _make_queue_empty(Queue *q);
+ void _delete_closed_sm(Queue *q);
+
+ // hooks for pre-warming pool size algorithm
+ void _prewarm_on_event_interval(const PreWarm::SPtrConstDst &dst, const Info &info);
+ void _prewarm_on_dequeue(const PreWarm::SPtrConstDst &dst, const Info &info);
+
+ ////
+ // Variables
+ //
+ PreWarm::Algorithm _algorithm = PreWarm::Algorithm::V1;
+
+ Event *_tick_event = nullptr;
+ ink_hrtime _event_period = HRTIME_SECONDS(1);
+
+ // Force PreWarmSM to open new netvc to keep the connection warm periodically
+ ActivityCop<PreWarmSM> _cop;
+ DLL<PreWarmSM> _cop_list;
+
+ Map _map;
+};
+
+/**
+ @class PreWarmManager
+ @details
+ - Global singleton object
+ - Responsible for stats & configs management
+ */
+class PreWarmManager
+{
+public:
+ static void reconfigure_prewarming_on_threads();
+
+ // Controllers
+ void start();
+ void reconfigure();
+ void stop();
+
+ // References
+ const PreWarm::ParsedSNIConf &get_parsed_conf() const;
+ const PreWarm::StatsIdMap &get_stats_id_map() const;
+
+ ////
+ // Variables
+ //
+ DynamicStats stats;
+
+private:
+ void _parse_sni_conf(PreWarm::ParsedSNIConf &parsed_conf, const SNIConfigParams *sni_conf) const;
+ void _register_stats(const PreWarm::ParsedSNIConf &parsed_conf);
+
+ ////
+ // Variables
+ //
+ // For the race of Main Thread (start up) vs Task Thread (config reload)
+ Ptr<ProxyMutex> _mutex;
+
+ PreWarm::ParsedSNIConf _parsed_conf;
+ PreWarm::StatsIdMap _stats_id_map;
+};
diff --git a/proxy/http/unit_tests/test_PreWarm.cc b/proxy/http/unit_tests/test_PreWarm.cc
new file mode 100644
index 0000000..3f22b89
--- /dev/null
+++ b/proxy/http/unit_tests/test_PreWarm.cc
@@ -0,0 +1,223 @@
+/** @file
+
+ Unit Tests for Pre-Warming Pool Size Algorithm
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "PreWarmAlgorithm.h"
+
+#define CATCH_CONFIG_MAIN
+#include "catch.hpp"
+
+TEST_CASE("PreWarm Algorithm", "[prewarm]")
+{
+ SECTION("prewarm_size_v1_on_event_interval")
+ {
+ SECTION("{min, max} = {10, 100}")
+ {
+ const uint32_t min = 10;
+ const uint32_t max = 100;
+
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 0, min, max) == min);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 0, min, max) == 20);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 0, min, max) == max);
+
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 5, min, max) == 5);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 5, min, max) == 15);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 5, min, max) == 95);
+
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 10, min, max) == 0);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 10, min, max) == 10);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 10, min, max) == 90);
+
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 50, min, max) == 0);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 50, min, max) == 0);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 50, min, max) == 50);
+ }
+
+ SECTION("{min, max} = {0, 0}")
+ {
+ const uint32_t min = 0;
+ const uint32_t max = 0;
+
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 0, min, max) == 0);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 0, min, max) == 0);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 0, min, max) == 0);
+
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 5, min, max) == 0);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 5, min, max) == 0);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 5, min, max) == 0);
+
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 10, min, max) == 0);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 10, min, max) == 0);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 10, min, max) == 0);
+
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 50, min, max) == 0);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 50, min, max) == 0);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 50, min, max) == 0);
+ }
+
+ SECTION("{min, max} = {10, -1}")
+ {
+ const uint32_t min = 10;
+ const uint32_t max = -1;
+
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 0, min, max) == min);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 0, min, max) == 20);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 0, min, max) == 101);
+
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 5, min, max) == 5);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 5, min, max) == 15);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 5, min, max) == 96);
+
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 10, min, max) == 0);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 10, min, max) == 10);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 10, min, max) == 91);
+
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(0, 50, min, max) == 0);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(20, 50, min, max) == 0);
+ CHECK(PreWarm::prewarm_size_v1_on_event_interval(101, 50, min, max) == 51);
+ }
+ }
+
+ SECTION("prewarm_size_v4_on_event_interval")
+ {
+ const uint32_t min = 10;
+ const uint32_t max = 100;
+
+ // same as v3
+ SECTION("rate = 1.0")
+ {
+ const double rate = 1.0;
+
+ SECTION("hit + miss + current_size < min ")
+ {
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 0, 1, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9);
+
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 1, 1, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 1, 1, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9);
+ }
+
+ SECTION("min <= miss + hit + current_size")
+ {
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 10, 10, min, max, rate) == 10);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 100, min, max, rate) == 0);
+
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 9, 90, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 90, min, max, rate) == 10);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 11, 90, min, max, rate) == 10);
+
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 9, 91, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 91, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 11, 91, min, max, rate) == 9);
+ }
+ }
+
+ SECTION("rate = 0.0")
+ {
+ const double rate = 0.0;
+
+ SECTION("hit + miss + current_size < min ")
+ {
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 0, 1, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9);
+
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 1, 1, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 1, 1, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9);
+ }
+
+ SECTION("min <= miss + hit + current_size")
+ {
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 10, 10, min, max, rate) == 0);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 100, min, max, rate) == 0);
+
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 9, 90, min, max, rate) == 0);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 90, min, max, rate) == 0);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 11, 90, min, max, rate) == 0);
+
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 9, 91, min, max, rate) == 0);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 91, min, max, rate) == 0);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 11, 91, min, max, rate) == 0);
+ }
+ }
+
+ SECTION("rate = 0.5")
+ {
+ const double rate = 0.5;
+
+ SECTION("hit + miss + current_size < min ")
+ {
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 0, 1, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9);
+
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 1, 1, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 1, 1, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9);
+ }
+
+ SECTION("min <= miss + hit + current_size")
+ {
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 10, 10, min, max, rate) == 5);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 100, min, max, rate) == 0);
+
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 9, 90, min, max, rate) == 4);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 90, min, max, rate) == 5);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 11, 90, min, max, rate) == 5);
+
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 18, 90, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 19, 90, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 20, 90, min, max, rate) == 10);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 21, 90, min, max, rate) == 10);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 22, 90, min, max, rate) == 10);
+ }
+ }
+
+ SECTION("rate = 1.5")
+ {
+ const double rate = 1.5;
+
+ SECTION("hit + miss + current_size < min ")
+ {
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 0, 1, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9);
+
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 1, 1, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 1, 1, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 0, 1, min, max, rate) == 9);
+ }
+
+ SECTION("min <= miss + hit + current_size")
+ {
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(0, 10, 10, min, max, rate) == 15);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 100, min, max, rate) == 0);
+
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 5, 90, min, max, rate) == 7);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 6, 90, min, max, rate) == 9);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 7, 90, min, max, rate) == 10);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 8, 90, min, max, rate) == 10);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 9, 90, min, max, rate) == 10);
+ CHECK(PreWarm::prewarm_size_v2_on_event_interval(1, 10, 90, min, max, rate) == 10);
+ }
+ }
+ }
+}
diff --git a/src/traffic_quic/traffic_quic.cc b/src/traffic_quic/traffic_quic.cc
index 4ee2f5f..8f42a9b 100644
--- a/src/traffic_quic/traffic_quic.cc
+++ b/src/traffic_quic/traffic_quic.cc
@@ -343,3 +343,11 @@ void
HttpCacheAction::cancel(Continuation *c)
{
}
+
+#include "PreWarmManager.h"
+void
+PreWarmManager::reconfigure()
+{
+}
+
+PreWarmManager prewarmManager;