You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by wk...@apache.org on 2023/03/01 02:24:01 UTC
[trafficserver] 01/04: Add 20 metrics for TCP connections created for tunnels.
This is an automated email from the ASF dual-hosted git repository.
wkaras pushed a commit to branch tunnel_stats
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 7818df5e3594c304816db0844cfdc4f67564cf50
Author: Walt Karas <wk...@yahooinc.com>
AuthorDate: Fri Feb 10 05:22:24 2023 +0000
Add 20 metrics for TCP connections created for tunnels.
Add current and total metrics for TCP connetions towards clients and servers, for blind TCP, TLS tunnel, TLS forward,
TLS partial blind tunnels, and tunnels over TLS created by an HTTP request (such as a websocket).
---
iocore/eventsystem/I_VConnection.h | 8 ++-
iocore/net/Net.cc | 54 ++++++++++++++---
iocore/net/P_Net.h | 20 ++++++
iocore/net/P_SSLNetVConnection.h | 3 +
iocore/net/P_UnixNetVConnection.h | 18 ++++++
iocore/net/SSLNetVConnection.cc | 97 ++++++++++++++++++++++++++++++
iocore/net/UnixNetVConnection.cc | 54 +++++++++++++++++
proxy/ProxyTransaction.cc | 8 +++
proxy/ProxyTransaction.h | 2 +
proxy/http/HttpSM.cc | 6 ++
tests/gold_tests/connect/connect.test.py | 15 ++++-
tests/gold_tests/connect/gold/metrics.gold | 2 +
tests/tools/stdout_wait | 81 +++++++++++++++++++++++++
13 files changed, 359 insertions(+), 9 deletions(-)
diff --git a/iocore/eventsystem/I_VConnection.h b/iocore/eventsystem/I_VConnection.h
index c60915e78..c656c040a 100644
--- a/iocore/eventsystem/I_VConnection.h
+++ b/iocore/eventsystem/I_VConnection.h
@@ -362,7 +362,13 @@ public:
return false;
}
-public:
+ // This function should be called when the VConnection is a tunnel endpoint. By default, a VConnection does not care if it
+ // is a tunnel endpoint.
+ virtual void
+ make_tunnel_endpoint()
+ {
+ }
+
/**
The error code from the last error.
diff --git a/iocore/net/Net.cc b/iocore/net/Net.cc
index 5d3e64fe4..14e508b15 100644
--- a/iocore/net/Net.cc
+++ b/iocore/net/Net.cc
@@ -98,13 +98,33 @@ register_net_stats()
};
const std::pair<const char *, Net_Stats> non_persistent[] = {
- {"proxy.process.net.accepts_currently_open", net_accepts_currently_open_stat },
- {"proxy.process.net.connections_currently_open", net_connections_currently_open_stat },
- {"proxy.process.net.default_inactivity_timeout_applied", default_inactivity_timeout_applied_stat},
- {"proxy.process.net.default_inactivity_timeout_count", default_inactivity_timeout_count_stat },
- {"proxy.process.net.dynamic_keep_alive_timeout_in_count", keep_alive_queue_timeout_count_stat },
- {"proxy.process.net.dynamic_keep_alive_timeout_in_total", keep_alive_queue_timeout_total_stat },
- {"proxy.process.socks.connections_currently_open", socks_connections_currently_open_stat },
+ {"proxy.process.net.accepts_currently_open", net_accepts_currently_open_stat },
+ {"proxy.process.net.connections_currently_open", net_connections_currently_open_stat },
+ {"proxy.process.net.default_inactivity_timeout_applied", default_inactivity_timeout_applied_stat },
+ {"proxy.process.net.default_inactivity_timeout_count", default_inactivity_timeout_count_stat },
+ {"proxy.process.net.dynamic_keep_alive_timeout_in_count", keep_alive_queue_timeout_count_stat },
+ {"proxy.process.net.dynamic_keep_alive_timeout_in_total", keep_alive_queue_timeout_total_stat },
+ {"proxy.process.socks.connections_currently_open", socks_connections_currently_open_stat },
+ {"proxy.process.tunnel.total_client_connections_blind_tcp", tunnel_total_client_connections_blind_tcp_stat },
+ {"proxy.process.tunnel.current_client_connections_blind_tcp", tunnel_current_client_connections_blind_tcp_stat },
+ {"proxy.process.tunnel.total_server_connections_blind_tcp", tunnel_total_server_connections_blind_tcp_stat },
+ {"proxy.process.tunnel.current_server_connections_blind_tcp", tunnel_current_server_connections_blind_tcp_stat },
+ {"proxy.process.tunnel.total_client_connections_tls_tunnel", tunnel_total_client_connections_tls_tunnel_stat },
+ {"proxy.process.tunnel.current_client_connections_tls_tunnel", tunnel_current_client_connections_tls_tunnel_stat },
+ {"proxy.process.tunnel.total_server_connections_tls_tunnel", tunnel_total_server_connections_tls_tunnel_stat },
+ {"proxy.process.tunnel.current_server_connections_tls_tunnel", tunnel_current_server_connections_tls_tunnel_stat },
+ {"proxy.process.tunnel.total_client_connections_tls_forward", tunnel_total_client_connections_tls_forward_stat },
+ {"proxy.process.tunnel.current_client_connections_tls_forward", tunnel_current_client_connections_tls_forward_stat },
+ {"proxy.process.tunnel.total_server_connections_tls_forward", tunnel_total_server_connections_tls_forward_stat },
+ {"proxy.process.tunnel.current_server_connections_tls_forward", tunnel_current_server_connections_tls_forward_stat },
+ {"proxy.process.tunnel.total_client_connections_tls_partial_blind", tunnel_total_client_connections_tls_partial_blind_stat },
+ {"proxy.process.tunnel.current_client_connections_tls_partial_blind", tunnel_current_client_connections_tls_partial_blind_stat},
+ {"proxy.process.tunnel.total_server_connections_tls_partial_blind", tunnel_total_server_connections_tls_partial_blind_stat },
+ {"proxy.process.tunnel.current_server_connections_tls_partial_blind", tunnel_current_server_connections_tls_partial_blind_stat},
+ {"proxy.process.tunnel.total_client_connections_tls_http", tunnel_total_client_connections_tls_http_stat },
+ {"proxy.process.tunnel.current_client_connections_tls_http", tunnel_current_client_connections_tls_http_stat },
+ {"proxy.process.tunnel.total_server_connections_tls_http", tunnel_total_server_connections_tls_http_stat },
+ {"proxy.process.tunnel.current_server_connections_tls_http", tunnel_current_server_connections_tls_http_stat },
};
for (auto &p : persistent) {
@@ -129,6 +149,26 @@ register_net_stats()
NET_CLEAR_DYN_STAT(keep_alive_queue_timeout_count_stat);
NET_CLEAR_DYN_STAT(default_inactivity_timeout_count_stat);
NET_CLEAR_DYN_STAT(default_inactivity_timeout_applied_stat);
+ NET_CLEAR_DYN_STAT(tunnel_total_client_connections_blind_tcp_stat);
+ NET_CLEAR_DYN_STAT(tunnel_current_client_connections_blind_tcp_stat);
+ NET_CLEAR_DYN_STAT(tunnel_total_server_connections_blind_tcp_stat);
+ NET_CLEAR_DYN_STAT(tunnel_current_server_connections_blind_tcp_stat);
+ NET_CLEAR_DYN_STAT(tunnel_total_client_connections_tls_tunnel_stat);
+ NET_CLEAR_DYN_STAT(tunnel_current_client_connections_tls_tunnel_stat);
+ NET_CLEAR_DYN_STAT(tunnel_total_server_connections_tls_tunnel_stat);
+ NET_CLEAR_DYN_STAT(tunnel_current_server_connections_tls_tunnel_stat);
+ NET_CLEAR_DYN_STAT(tunnel_total_client_connections_tls_forward_stat);
+ NET_CLEAR_DYN_STAT(tunnel_current_client_connections_tls_forward_stat);
+ NET_CLEAR_DYN_STAT(tunnel_total_server_connections_tls_forward_stat);
+ NET_CLEAR_DYN_STAT(tunnel_current_server_connections_tls_forward_stat);
+ NET_CLEAR_DYN_STAT(tunnel_total_client_connections_tls_partial_blind_stat);
+ NET_CLEAR_DYN_STAT(tunnel_current_client_connections_tls_partial_blind_stat);
+ NET_CLEAR_DYN_STAT(tunnel_total_server_connections_tls_partial_blind_stat);
+ NET_CLEAR_DYN_STAT(tunnel_current_server_connections_tls_partial_blind_stat);
+ NET_CLEAR_DYN_STAT(tunnel_total_client_connections_tls_http_stat);
+ NET_CLEAR_DYN_STAT(tunnel_current_client_connections_tls_http_stat);
+ NET_CLEAR_DYN_STAT(tunnel_total_server_connections_tls_http_stat);
+ NET_CLEAR_DYN_STAT(tunnel_current_server_connections_tls_http_stat);
RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.tcp.total_accepts", RECD_INT, RECP_NON_PERSISTENT,
static_cast<int>(net_tcp_accept_stat), RecRawStatSyncSum);
diff --git a/iocore/net/P_Net.h b/iocore/net/P_Net.h
index 09c2fa125..dfce6064b 100644
--- a/iocore/net/P_Net.h
+++ b/iocore/net/P_Net.h
@@ -57,6 +57,26 @@ enum Net_Stats {
net_connections_throttled_in_stat,
net_connections_throttled_out_stat,
net_requests_max_throttled_in_stat,
+ tunnel_total_client_connections_blind_tcp_stat,
+ tunnel_current_client_connections_blind_tcp_stat,
+ tunnel_total_server_connections_blind_tcp_stat,
+ tunnel_current_server_connections_blind_tcp_stat,
+ tunnel_total_client_connections_tls_tunnel_stat,
+ tunnel_current_client_connections_tls_tunnel_stat,
+ tunnel_total_server_connections_tls_tunnel_stat,
+ tunnel_current_server_connections_tls_tunnel_stat,
+ tunnel_total_client_connections_tls_forward_stat,
+ tunnel_current_client_connections_tls_forward_stat,
+ tunnel_total_server_connections_tls_forward_stat,
+ tunnel_current_server_connections_tls_forward_stat,
+ tunnel_total_client_connections_tls_partial_blind_stat,
+ tunnel_current_client_connections_tls_partial_blind_stat,
+ tunnel_total_server_connections_tls_partial_blind_stat,
+ tunnel_current_server_connections_tls_partial_blind_stat,
+ tunnel_total_client_connections_tls_http_stat,
+ tunnel_current_client_connections_tls_http_stat,
+ tunnel_total_server_connections_tls_http_stat,
+ tunnel_current_server_connections_tls_http_stat,
Net_Stat_Count
};
diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h
index ca6f8b17a..babc45ecd 100644
--- a/iocore/net/P_SSLNetVConnection.h
+++ b/iocore/net/P_SSLNetVConnection.h
@@ -476,6 +476,9 @@ private:
ssl_error_t _ssl_write_buffer(const void *buf, int64_t nbytes, int64_t &nwritten);
ssl_error_t _ssl_connect();
ssl_error_t _ssl_accept();
+
+ void _in_context_tunnel() override;
+ void _out_context_tunnel() override;
};
typedef int (SSLNetVConnection::*SSLNetVConnHandler)(int, void *);
diff --git a/iocore/net/P_UnixNetVConnection.h b/iocore/net/P_UnixNetVConnection.h
index 3b5647bbe..35e4ee4d5 100644
--- a/iocore/net/P_UnixNetVConnection.h
+++ b/iocore/net/P_UnixNetVConnection.h
@@ -44,6 +44,8 @@ struct PollDescriptor;
enum tcp_congestion_control_t { CLIENT_SIDE, SERVER_SIDE };
+// WARNING: many or most of the member functions of UnixNetVConnection should only be used when it is instantiated
+// directly. They should not be used when UnixNetVConnection is a base class.
class UnixNetVConnection : public NetVConnection, public NetEvent
{
public:
@@ -225,9 +227,25 @@ public:
friend void write_to_net_io(NetHandler *, UnixNetVConnection *, EThread *);
+ // set_context() should be called before calling this member function.
+ void make_tunnel_endpoint() override;
+
+ bool
+ is_tunnel_endpoint() const
+ {
+ return _is_tunnel_endpoint;
+ }
+
private:
virtual void *_prepareForMigration();
virtual NetProcessor *_getNetProcessor();
+ bool _is_tunnel_endpoint{false};
+
+ // Called by make_tunnel_endpiont() when the far end of the TCP connection is the active/client end.
+ virtual void _in_context_tunnel();
+
+ // Called by make_tunnel_endpiont() when the far end of the TCP connection is the passive/server end.
+ virtual void _out_context_tunnel();
};
extern ClassAllocator<UnixNetVConnection> netVCAllocator;
diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc
index cfa44477e..3a3c648e9 100644
--- a/iocore/net/SSLNetVConnection.cc
+++ b/iocore/net/SSLNetVConnection.cc
@@ -1001,6 +1001,45 @@ SSLNetVConnection::free(EThread *t)
}
con.close();
+ if (is_tunnel_endpoint()) {
+ ink_assert(get_context() != NET_VCONNECTION_UNSET);
+
+ int c;
+
+ if (get_context() == NET_VCONNECTION_IN) {
+ switch (get_tunnel_type()) {
+ case SNIRoutingType::BLIND:
+ c = tunnel_current_client_connections_tls_tunnel_stat;
+ break;
+ case SNIRoutingType::FORWARD:
+ c = tunnel_current_client_connections_tls_forward_stat;
+ break;
+ case SNIRoutingType::PARTIAL_BLIND:
+ c = tunnel_current_client_connections_tls_partial_blind_stat;
+ break;
+ default:
+ c = tunnel_current_client_connections_tls_http_stat;
+ break;
+ }
+ } else { // NET_VCONNECTION_OUT
+ switch (get_tunnel_type()) {
+ case SNIRoutingType::BLIND:
+ c = tunnel_current_server_connections_tls_tunnel_stat;
+ break;
+ case SNIRoutingType::FORWARD:
+ c = tunnel_current_server_connections_tls_forward_stat;
+ break;
+ case SNIRoutingType::PARTIAL_BLIND:
+ c = tunnel_current_server_connections_tls_partial_blind_stat;
+ break;
+ default:
+ c = tunnel_current_server_connections_tls_http_stat;
+ break;
+ }
+ }
+ NET_DECREMENT_DYN_STAT(c);
+ }
+
#if TS_HAS_TLS_EARLY_DATA
if (_early_data_reader != nullptr) {
_early_data_reader->dealloc();
@@ -1913,6 +1952,64 @@ SSLNetVConnection::populate(Connection &con, Continuation *c, void *arg)
return EVENT_DONE;
}
+void
+SSLNetVConnection::_in_context_tunnel()
+{
+ ink_assert(get_context() == NET_VCONNECTION_IN);
+
+ int t, c;
+
+ switch (get_tunnel_type()) {
+ case SNIRoutingType::BLIND:
+ t = tunnel_total_client_connections_tls_tunnel_stat;
+ c = tunnel_current_client_connections_tls_tunnel_stat;
+ break;
+ case SNIRoutingType::FORWARD:
+ t = tunnel_total_client_connections_tls_forward_stat;
+ c = tunnel_current_client_connections_tls_forward_stat;
+ break;
+ case SNIRoutingType::PARTIAL_BLIND:
+ t = tunnel_total_client_connections_tls_partial_blind_stat;
+ c = tunnel_current_client_connections_tls_partial_blind_stat;
+ break;
+ default:
+ t = tunnel_total_client_connections_tls_http_stat;
+ c = tunnel_current_client_connections_tls_http_stat;
+ break;
+ }
+ NET_INCREMENT_DYN_STAT(t);
+ NET_INCREMENT_DYN_STAT(c);
+}
+
+void
+SSLNetVConnection::_out_context_tunnel()
+{
+ ink_assert(get_context() == NET_VCONNECTION_OUT);
+
+ int t, c;
+
+ switch (get_tunnel_type()) {
+ case SNIRoutingType::BLIND:
+ t = tunnel_total_server_connections_tls_tunnel_stat;
+ c = tunnel_current_server_connections_tls_tunnel_stat;
+ break;
+ case SNIRoutingType::FORWARD:
+ t = tunnel_total_server_connections_tls_forward_stat;
+ c = tunnel_current_server_connections_tls_forward_stat;
+ break;
+ case SNIRoutingType::PARTIAL_BLIND:
+ t = tunnel_total_server_connections_tls_partial_blind_stat;
+ c = tunnel_current_server_connections_tls_partial_blind_stat;
+ break;
+ default:
+ t = tunnel_total_server_connections_tls_http_stat;
+ c = tunnel_current_server_connections_tls_http_stat;
+ break;
+ }
+ NET_INCREMENT_DYN_STAT(t);
+ NET_INCREMENT_DYN_STAT(c);
+}
+
void
SSLNetVConnection::increment_ssl_version_metric(int version) const
{
diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc
index f8a7d78d2..002c75632 100644
--- a/iocore/net/UnixNetVConnection.cc
+++ b/iocore/net/UnixNetVConnection.cc
@@ -1284,6 +1284,8 @@ UnixNetVConnection::clear()
void
UnixNetVConnection::free(EThread *t)
{
+ Debug("iocore_net", "Entering UnixNetVConnection::free()");
+
ink_release_assert(t == this_ethread());
// close socket fd
@@ -1292,6 +1294,23 @@ UnixNetVConnection::free(EThread *t)
}
con.close();
+ if (is_tunnel_endpoint()) {
+ Debug("iocore_net", "Freeing UnixNetVConnection that is tunnel endpoint");
+
+ int c;
+ switch (get_context()) {
+ case NET_VCONNECTION_IN:
+ c = tunnel_current_client_connections_blind_tcp_stat;
+ break;
+ case NET_VCONNECTION_OUT:
+ c = tunnel_current_server_connections_blind_tcp_stat;
+ break;
+ default:
+ ink_release_assert(false);
+ }
+ NET_DECREMENT_DYN_STAT(c);
+ }
+
clear();
SET_CONTINUATION_HANDLER(this, &UnixNetVConnection::startEvent);
ink_assert(con.fd == NO_FD);
@@ -1491,3 +1510,38 @@ UnixNetVConnection::set_tcp_congestion_control(int side)
return -1;
#endif
}
+
+void
+UnixNetVConnection::make_tunnel_endpoint()
+{
+ Debug("iocore_net", "Entering UnixNetVConnection::make_tunnel_endpoint()");
+
+ ink_assert(!_is_tunnel_endpoint);
+
+ _is_tunnel_endpoint = true;
+
+ switch (get_context()) {
+ case NET_VCONNECTION_IN:
+ _in_context_tunnel();
+ break;
+ case NET_VCONNECTION_OUT:
+ _out_context_tunnel();
+ break;
+ default:
+ ink_release_assert(false);
+ }
+}
+
+void
+UnixNetVConnection::_in_context_tunnel()
+{
+ NET_INCREMENT_DYN_STAT(tunnel_total_client_connections_blind_tcp_stat);
+ NET_INCREMENT_DYN_STAT(tunnel_current_client_connections_blind_tcp_stat);
+}
+
+void
+UnixNetVConnection::_out_context_tunnel()
+{
+ NET_INCREMENT_DYN_STAT(tunnel_total_server_connections_blind_tcp_stat);
+ NET_INCREMENT_DYN_STAT(tunnel_current_server_connections_blind_tcp_stat);
+}
diff --git a/proxy/ProxyTransaction.cc b/proxy/ProxyTransaction.cc
index cb80b3cc2..66247db3c 100644
--- a/proxy/ProxyTransaction.cc
+++ b/proxy/ProxyTransaction.cc
@@ -248,3 +248,11 @@ void
ProxyTransaction::set_close_connection(HTTPHdr &hdr) const
{
}
+
+void
+ProxyTransaction::make_tunnel_endpoint()
+{
+ auto nvc = get_netvc();
+ ink_assert(nvc != nullptr);
+ nvc->make_tunnel_endpoint();
+}
diff --git a/proxy/ProxyTransaction.h b/proxy/ProxyTransaction.h
index c6d1ba79c..806105e56 100644
--- a/proxy/ProxyTransaction.h
+++ b/proxy/ProxyTransaction.h
@@ -130,6 +130,8 @@ public:
bool support_sni() const;
+ void make_tunnel_endpoint() override;
+
/// Variables
//
HttpSessionAccept::Options upstream_outbound_options; // overwritable copy of options
diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc
index e478ba67c..8598fa61b 100644
--- a/proxy/http/HttpSM.cc
+++ b/proxy/http/HttpSM.cc
@@ -6968,6 +6968,9 @@ HttpSM::setup_push_transfer_to_cache()
void
HttpSM::setup_blind_tunnel(bool send_response_hdr, IOBufferReader *initial)
{
+ ink_assert(server_entry->vc != nullptr);
+ ink_assert(ua_entry->vc != nullptr);
+
HttpTunnelConsumer *c_ua;
HttpTunnelConsumer *c_os;
HttpTunnelProducer *p_ua;
@@ -7025,6 +7028,9 @@ HttpSM::setup_blind_tunnel(bool send_response_hdr, IOBufferReader *initial)
c_os = tunnel.add_consumer(server_entry->vc, ua_entry->vc, &HttpSM::tunnel_handler_ssl_consumer, HT_HTTP_SERVER,
"http server - tunnel");
+ ua_entry->vc->make_tunnel_endpoint();
+ server_entry->vc->make_tunnel_endpoint();
+
// Make the tunnel aware that the entries are bi-directional
tunnel.chain(c_os, p_os);
tunnel.chain(c_ua, p_ua);
diff --git a/tests/gold_tests/connect/connect.test.py b/tests/gold_tests/connect/connect.test.py
index 95b25958a..60a5a32e0 100644
--- a/tests/gold_tests/connect/connect.test.py
+++ b/tests/gold_tests/connect/connect.test.py
@@ -135,7 +135,7 @@ class ConnectViaPVTest:
self.ts.Disk.records_config.update({
'proxy.config.diags.debug.enabled': 1,
- 'proxy.config.diags.debug.tags': 'http',
+ 'proxy.config.diags.debug.tags': 'http|iocore_net|rec',
'proxy.config.http.server_ports': f"{self.ts.Variables.port}",
'proxy.config.http.connect_ports': f"{self.server.Variables.http_port}",
})
@@ -160,8 +160,21 @@ class ConnectViaPVTest:
tr.StillRunningAfter = self.server
tr.StillRunningAfter = self.ts
+ def __testTunnelMetrics(self):
+ tr = Test.AddTestRun("Reload config")
+ tr.Processes.Default.Command = (
+ f"{Test.Variables.AtsTestToolsDir}/stdout_wait " +
+ "'traffic_ctl metric get proxy.process.tunnel.total_client_connections_blind_tcp" +
+ f" proxy.process.tunnel.total_server_connections_blind_tcp' {Test.TestDirectory}/gold/metrics.gold"
+ )
+ # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket
+ tr.Processes.Default.Env = self.ts.Env
+ tr.Processes.Default.ReturnCode = 0
+ tr.StillRunningAfter = self.ts
+
def run(self):
self.runTraffic()
+ self.__testTunnelMetrics()
ConnectViaPVTest().run()
diff --git a/tests/gold_tests/connect/gold/metrics.gold b/tests/gold_tests/connect/gold/metrics.gold
new file mode 100644
index 000000000..8125abaf8
--- /dev/null
+++ b/tests/gold_tests/connect/gold/metrics.gold
@@ -0,0 +1,2 @@
+proxy.process.tunnel.total_client_connections_blind_tcp 1
+proxy.process.tunnel.total_server_connections_blind_tcp 1
diff --git a/tests/tools/stdout_wait b/tests/tools/stdout_wait
new file mode 100755
index 000000000..08c23d6db
--- /dev/null
+++ b/tests/tools/stdout_wait
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A bash script to repeatedly run a command until its standard output matches a gold file.
+#
+# usage:
+# stdout_wait [ MAX-WAIT [ POST-WAIT ] ] COMMAND GOLD-FILE-SPEC
+#
+# (COMMAND may contain white space.)
+#
+# MAX-WAIT is the maximum number of seconds to wait for the condition. If it is omitted, it defaults to 60.
+#
+# POST-WAIT is the number of seconds to wait after the condition is true. If it is omitted, it defaults to 0.
+#
+# The script exits with status 0 when the command's standard output matches the gold file. It exits with status 1 if the
+# output never matche the gold file, and the maximum wait has expired.
+
+WAIT=60
+POST_WAIT=0
+
+USAGE="usage: stdout_wait [ MAX-WAIT [ POST-WAIT ] ] COMMAND GOLD-FILE-SPEC"
+
+if [[ "$1" = "" ]] ; then
+ echo $USAGE >&2
+ exit 1
+fi
+
+X=$( echo "$1" | sed 's/x/yy/g' | sed 's/[^0-9]/x/g' )
+if [[ "$X" = "$1" ]] ; then
+ WAIT=$1
+ shift
+ if [[ "$1" = "" ]] ; then
+ echo $USAGE >&2
+ exit 1
+ fi
+ X=$( echo "$1" | sed 's/x/yy/g' | sed 's/[^0-9]/x/g' )
+ if [[ "$X" = "$1" ]] ; then
+ POST_WAIT=$1
+ shift
+ fi
+fi
+
+if [[ "$1" = "" ]] ; then
+ echo $USAGE >&2
+ exit 1
+fi
+
+if [[ ! -f "$2" ]] ; then
+ echo $USAGE >&2
+ exit 1
+fi
+
+while (( WAIT > 0 ))
+do
+ if $1 | diff - $2 > /dev/null 2> /dev/null
+ then
+ if (( POST_WAIT > 0 ))
+ then
+ sleep $POST_WAIT
+ fi
+ exit 0
+ fi
+ sleep 1
+ let WAIT=WAIT-1
+done
+exit 1