You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by zw...@apache.org on 2022/04/05 20:08:57 UTC
[trafficserver] branch 9.2.x updated: STEK share plugin using Raft (#8751)
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch 9.2.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/9.2.x by this push:
new dfda50355 STEK share plugin using Raft (#8751)
dfda50355 is described below
commit dfda503557ca9e79482fedd24a6deea9ba701c0c
Author: Fei Deng <du...@gmail.com>
AuthorDate: Mon Apr 4 13:46:17 2022 -0500
STEK share plugin using Raft (#8751)
Co-authored-by: Fei Deng <fe...@yahooinc.com>
(cherry picked from commit dd3f1c30be4f395afe0aa3996a3ea1ac7dd4431f)
---
build/hiredis.m4 | 2 +-
build/nuraft.m4 | 85 ++++
configure.ac | 15 +-
doc/admin-guide/plugins/index.en.rst | 4 +
doc/admin-guide/plugins/stek_share.en.rst | 105 +++++
plugins/Makefile.am | 4 +
plugins/experimental/stek_share/Makefile.inc | 36 ++
plugins/experimental/stek_share/common.cc | 46 +++
plugins/experimental/stek_share/common.h | 72 ++++
.../stek_share/example_server_conf.yaml | 16 +
.../stek_share/example_server_list.yaml | 15 +
plugins/experimental/stek_share/log_store.cc | 263 ++++++++++++
plugins/experimental/stek_share/log_store.h | 76 ++++
plugins/experimental/stek_share/state_machine.h | 237 +++++++++++
plugins/experimental/stek_share/state_manager.h | 102 +++++
plugins/experimental/stek_share/stek_share.cc | 445 +++++++++++++++++++++
plugins/experimental/stek_share/stek_share.h | 123 ++++++
plugins/experimental/stek_share/stek_utils.cc | 82 ++++
plugins/experimental/stek_share/stek_utils.h | 43 ++
.../pluginTest/stek_share/server_list.yaml | 15 +
.../pluginTest/stek_share/ssl/self_signed.crt | 32 ++
.../pluginTest/stek_share/ssl/self_signed.key | 52 +++
.../pluginTest/stek_share/stek_share.test.py | 254 ++++++++++++
23 files changed, 2118 insertions(+), 6 deletions(-)
diff --git a/build/hiredis.m4 b/build/hiredis.m4
index 1a9a18c02..b2f09179b 100644
--- a/build/hiredis.m4
+++ b/build/hiredis.m4
@@ -34,7 +34,7 @@ AC_ARG_WITH(hiredis, [AS_HELP_STRING([--with-hiredis=DIR],[use a specific hiredi
case "$hiredis_base_dir" in
*":"*)
- hidredis_include="`echo $hiredis_base_dir |sed -e 's/:.*$//'`"
+ hiredis_include="`echo $hiredis_base_dir |sed -e 's/:.*$//'`"
hiredis_ldflags="`echo $hiredis_base_dir |sed -e 's/^.*://'`"
AC_MSG_CHECKING(for hiredis includes in $hiredis_include libs in $hiredis_ldflags )
;;
diff --git a/build/nuraft.m4 b/build/nuraft.m4
new file mode 100644
index 000000000..dca9e96a9
--- /dev/null
+++ b/build/nuraft.m4
@@ -0,0 +1,85 @@
+dnl -------------------------------------------------------- -*- autoconf -*-
+dnl Licensed to the Apache Software Foundation (ASF) under one or more
+dnl contributor license agreements. See the NOTICE file distributed with
+dnl this work for additional information regarding copyright ownership.
+dnl The ASF licenses this file to You under the Apache License, Version 2.0
+dnl (the "License"); you may not use this file except in compliance with
+dnl the License. You may obtain a copy of the License at
+dnl
+dnl http://www.apache.org/licenses/LICENSE-2.0
+dnl
+dnl Unless required by applicable law or agreed to in writing, software
+dnl distributed under the License is distributed on an "AS IS" BASIS,
+dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+dnl See the License for the specific language governing permissions and
+dnl limitations under the License.
+
+dnl
+dnl nuraft.m4: Trafficserver's nuraft autoconf macros
+dnl
+
+dnl
+dnl TS_CHECK_NURAFT: look for nuraft libraries and headers
+dnl
+
+AC_DEFUN([TS_CHECK_NURAFT], [
+has_nuraft=no
+AC_ARG_WITH(nuraft, [AC_HELP_STRING([--with-nuraft=DIR], [use a specific nuraft library])],
+[
+ if test "x$withval" != "xyes" && test "x$withval" != "x"; then
+ nuraft_base_dir="$withval"
+ if test "$withval" != "no"; then
+ has_nuraft=yes
+ case "$withval" in
+ *":"*)
+ nuraft_include="`echo $withval | sed -e 's/:.*$//'`"
+ nuraft_ldflags="`echo $withval | sed -e 's/^.*://'`"
+ AC_MSG_CHECKING(for nuraft includes in $nuraft_include libs in $nuraft_ldflags)
+ ;;
+ *)
+ nuraft_include="$withval/include"
+ nuraft_ldflags="$withval/lib"
+ nuraft_base_dir="$withval"
+ AC_MSG_CHECKING(for nuraft includes in $nuraft_include libs in $nuraft_ldflags)
+ ;;
+ esac
+ fi
+ fi
+
+ if test -d $nuraft_include && test -d $nuraft_ldflags && test -f $nuraft_include/libnuraft/nuraft.hxx; then
+ AC_MSG_RESULT([ok])
+ else
+ AC_MSG_RESULT([not found])
+ fi
+
+if test "$has_nuraft" != "no"; then
+ saved_ldflags=$LDFLAGS
+ saved_cppflags=$CPPFLAGS
+
+ NURAFT_LIBS=-lnuraft
+ if test "$nuraft_base_dir" != "/usr"; then
+ NURAFT_INCLUDES=-I${nuraft_include}
+ NURAFT_LDFLAGS=-L${nuraft_ldflags}
+
+ TS_ADDTO(CPPFLAGS, [${NURAFT_INCLUDES}])
+ TS_ADDTO(LDFLAGS, [${NURAFT_LDFLAGS}])
+ TS_ADDTO_RPATH(${nuraft_ldflags})
+ fi
+
+ if test "$nuraft_include" != "0"; then
+ NURAFT_INCLUDES=-I${nuraft_include}
+ else
+ has_nuraft=no
+ CPPFLAGS=$saved_cppflags
+ LDFLAGS=$saved_ldflags
+ fi
+fi
+],
+[
+ has_nuraft=no
+])
+
+AC_SUBST([NURAFT_INCLUDES])
+AC_SUBST([NURAFT_LIBS])
+AC_SUBST([NURAFT_LDFLAGS])
+])
diff --git a/configure.ac b/configure.ac
index 849884485..af8aba872 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1460,9 +1460,12 @@ TS_CHECK_BORINGOCSP
# Check for optional hiredis library
TS_CHECK_HIREDIS
-
AM_CONDITIONAL([BUILD_SSL_SESSION_REUSE_PLUGIN], [test ! -z "${LIB_HIREDIS}" -a "x${has_hiredis}" = "x1" ])
+# Check for optional nuraft library
+TS_CHECK_NURAFT
+AM_CONDITIONAL([BUILD_STEK_SHARE_PLUGIN], [test x"$has_nuraft" = x"yes"])
+
# Check for backtrace() support
has_backtrace=0
AC_CHECK_HEADERS([execinfo.h], [has_backtrace=1],[])
@@ -2320,13 +2323,15 @@ AC_MSG_NOTICE([Build option summary:
CXXFLAGS: $CXXFLAGS
CPPFLAGS: $CPPFLAGS
LDFLAGS: $LDFLAGS
- AM@&t@_CFLAGS: $AM_CFLAGS
- AM@&t@_CXXFLAGS: $AM_CXXFLAGS
- AM@&t@_CPPFLAGS: $AM_CPPFLAGS
- AM@&t@_LDFLAGS: $AM_LDFLAGS
+ AM@&t@_CFLAGS: $AM_CFLAGS
+ AM@&t@_CXXFLAGS: $AM_CXXFLAGS
+ AM@&t@_CPPFLAGS: $AM_CPPFLAGS
+ AM@&t@_LDFLAGS: $AM_LDFLAGS
TS_INCLUDES: $TS_INCLUDES
OPENSSL_LDFLAGS: $OPENSSL_LDFLAGS
OPENSSL_INCLUDES: $OPENSSL_INCLUDES
YAMLCPP_LDFLAGS: $YAMLCPP_LDFLAGS
YAMLCPP_INCLUDES: $YAMLCPP_INCLUDES
+ NURAFT_LDFLAGS: $NURAFT_LDFLAGS
+ NURAFT_INCLUDES: $NURAFT_INCLUDES
])
diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst
index a84847573..12ff6ad50 100644
--- a/doc/admin-guide/plugins/index.en.rst
+++ b/doc/admin-guide/plugins/index.en.rst
@@ -172,6 +172,7 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi
Slice <slice.en>
SSL Headers <sslheaders.en>
SSL Session Reuse <ssl_session_reuse.en>
+ STEK Share <stek_share.en>
System Statistics <system_stats.en>
Traffic Dump <traffic_dump.en>
WebP Transform <webp_transform.en>
@@ -265,6 +266,9 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi
:doc:`SSL Headers <sslheaders.en>`
Populate request headers with SSL session information.
+:doc:`STEK Share <stek_share.en>`
+ Coordinates STEK (Session Ticket Encryption Key) between ATS instances running in a group.
+
:doc:`System Stats <system_stats.en>`
Inserts system statistics in to the stats list
diff --git a/doc/admin-guide/plugins/stek_share.en.rst b/doc/admin-guide/plugins/stek_share.en.rst
new file mode 100644
index 000000000..43ac28c67
--- /dev/null
+++ b/doc/admin-guide/plugins/stek_share.en.rst
@@ -0,0 +1,105 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+
+.. include:: ../../common.defs
+
+.. _admin-plugins-stek_share:
+
+
+STEK Share Plugin
+*****************
+
+This plugin coordinates STEK (Session Ticket Encryption Key) between ATS instances running in a group.
+As the ID based session resumption rate continue to decrease, this new plugin will replace the
+:ref:`admin-plugins-ssl_session_reuse` plugin.
+
+
+How It Works
+============
+
+This plugin implements the `Raft consensus algorithm <https://raft.github.io/>` to decide on a leader. The leader will
+periodically create a new STEK key and share it with all other ATS boxes in the group. When the plugin starts up, it
+will automatically join the cluster of all other ATS boxes in the group, which will also automatically elect a leader.
+The plugin uses the `TSSslTicketKeyUpdate` call to update ATS with the latest two STEK's it has received.
+
+All communication are encrypted. All the ATS boxes participating in the STEK sharing must have access to the cert/key pair.
+
+Note that since the this plugin only updates STEK every few hours, all Raft related stuff are kept in memory, and some code is
+borrowed from the examples from `NuRaft library <https://github.com/eBay/NuRaft>` that is used in this plugin.
+
+
+Building
+========
+
+This plugin uses `NuRaft library <https://github.com/eBay/NuRaft>` for leader election and communication.
+The NuRaft library must be installed for this plugin to build. It can be specified by the `--with-nuraft` argument to configure.
+
+This plugin also uses `YAML-CPP library <https://github.com/jbeder/yaml-cpp>` for reading the configuration file.
+The YAML-CPP library must be installed for this plugin to build. It can be specified by the `--with-yaml-cpp` argument to configure.
+
+As part of the experimental plugs, the `--enable-experimental-plugins` option must also be given to configure to build this plugin.
+
+
+Config File
+===========
+
+STEK Share is a global plugin. Its configuration file uses YAML, and is given as an argument to the plugin in :file:`plugin.config`.
+
+::
+ stek_share.so etc/trafficserver/example_server_conf.yaml
+
+Available options:
+
+* server_id - An unique ID for the server.
+* address - Hostname or IP address of the server.
+* port - Port number for communication.
+* asio_thread_pool_size - [Optional] Thread pool size for `ASIO library <http://think-async.com/Asio/>`. Default size is 4.
+* heart_beat_interval - [Optional] Heart beat interval of Raft leader, must be less than "election_timeout_lower_bound". Default value is 100 ms.
+* election_timeout_lower_bound - [Optional] Lower bound of Raft leader election timeout. Default value is 200 ms.
+* election_timeout_upper_bound - [Optional] Upper bound of Raft leader election timeout. Default value is 400 ms.
+* reserved_log_items - [Optional] The maximum number of logs preserved ahead the last snapshot. Default value is 5.
+* snapshot_distance - [Optional] The number of log appends for each snapshot. Default value is 5.
+* client_req_timeout - [Optional] Client request timeout. Default value is 3000 ms.
+* key_update_interval - The interval between STEK update.
+* server_list_file - Path to a file containing information of all the servers that's supposed to be in the Raft cluster.
+* root_cert_file - Path to the root ca file.
+* server_cert_file - Path to the cert file.
+* server_key_file - Path to the key file.
+* cert_verify_str - SSL verification string, for example "/C=US/ST=IL/O=Yahoo/OU=Edge/CN=localhost"
+
+
+Example Config File
+===================
+
+.. literalinclude:: ../../../plugins/experimental/stek_share/example_server_conf.yaml
+
+
+Server List File
+================
+
+Server list file as mentioned above, also in YAML.
+
+* server_id - ID of the server.
+* address - Hostname or IP address of the server.
+* port - Port number of the server.
+
+
+Example Server List File
+========================
+
+.. literalinclude:: ../../../plugins/experimental/stek_share/example_server_list.yaml
diff --git a/plugins/Makefile.am b/plugins/Makefile.am
index 75876ae3e..fff6fdbd6 100644
--- a/plugins/Makefile.am
+++ b/plugins/Makefile.am
@@ -107,6 +107,10 @@ if BUILD_SSL_SESSION_REUSE_PLUGIN
include experimental/ssl_session_reuse/Makefile.inc
endif
+if BUILD_STEK_SHARE_PLUGIN
+include experimental/stek_share/Makefile.inc
+endif
+
if HAS_KYOTOCABINET
include experimental/cache_key_genid/Makefile.inc
endif
diff --git a/plugins/experimental/stek_share/Makefile.inc b/plugins/experimental/stek_share/Makefile.inc
new file mode 100644
index 000000000..462916d33
--- /dev/null
+++ b/plugins/experimental/stek_share/Makefile.inc
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Only build if NURAFT_LIBS is set to a non-empty value
+pkglib_LTLIBRARIES += experimental/stek_share/stek_share.la
+
+experimental_stek_share_stek_share_la_SOURCES = \
+ experimental/stek_share/common.cc \
+ experimental/stek_share/common.h \
+ experimental/stek_share/log_store.cc \
+ experimental/stek_share/log_store.h \
+ experimental/stek_share/state_machine.h \
+ experimental/stek_share/state_manager.h \
+ experimental/stek_share/stek_share.cc \
+ experimental/stek_share/stek_share.h \
+ experimental/stek_share/stek_utils.cc \
+ experimental/stek_share/stek_utils.h
+
+experimental_stek_share_stek_share_la_LDFLAGS = $(AM_LDFLAGS) @YAMLCPP_LDFLAGS@
+
+AM_CPPFLAGS += @NURAFT_INCLUDES@ @YAMLCPP_INCLUDES@
+
+experimental_stek_share_stek_share_la_LIBADD = @NURAFT_LIBS@ @YAMLCPP_LIBS@
diff --git a/plugins/experimental/stek_share/common.cc b/plugins/experimental/stek_share/common.cc
new file mode 100644
index 000000000..b9af195e7
--- /dev/null
+++ b/plugins/experimental/stek_share/common.cc
@@ -0,0 +1,46 @@
+/** @file
+
+ common.cc - Some common functions everyone needs
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ */
+
+#include <cstring>
+#include <openssl/ssl.h>
+
+#include <ts/ts.h>
+#include <ts/apidefs.h>
+
+#include "common.h"
+
+const unsigned char hex_chars[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'};
+
+std::string
+hex_str(std::string const &str)
+{
+ std::string hex_str;
+ hex_str.reserve(str.size() * 2);
+ for (unsigned long int i = 0; i < str.size(); ++i) {
+ unsigned char c = str.at(i);
+ hex_str[i * 2] = hex_chars[(c & 0xF0) >> 4];
+ hex_str[i * 2 + 1] = hex_chars[(c & 0x0F)];
+ }
+ return hex_str;
+}
diff --git a/plugins/experimental/stek_share/common.h b/plugins/experimental/stek_share/common.h
new file mode 100644
index 000000000..312ad557d
--- /dev/null
+++ b/plugins/experimental/stek_share/common.h
@@ -0,0 +1,72 @@
+/** @file
+
+ common.h - Things that need to be everywhere
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ */
+
+#pragma once
+
+#include <iostream>
+#include <iomanip>
+#include <string>
+#include <cstring>
+#include <mutex>
+#include <deque>
+#include <cmath>
+
+#define PLUGIN "stek_share"
+
+class PluginThreads
+{
+public:
+ void
+ store(const pthread_t &th)
+ {
+ std::lock_guard<std::mutex> lock(threads_mutex);
+ threads_queue.push_back(th);
+ }
+
+ void
+ terminate()
+ {
+ shut_down = true;
+
+ std::lock_guard<std::mutex> lock(threads_mutex);
+ while (!threads_queue.empty()) {
+ pthread_t th = threads_queue.front();
+ ::pthread_join(th, nullptr);
+ threads_queue.pop_front();
+ }
+ }
+
+ bool
+ is_shut_down()
+ {
+ return shut_down;
+ }
+
+private:
+ bool shut_down = false;
+ std::deque<pthread_t> threads_queue;
+ std::mutex threads_mutex;
+};
+
+std::string hex_str(std::string const &str);
diff --git a/plugins/experimental/stek_share/example_server_conf.yaml b/plugins/experimental/stek_share/example_server_conf.yaml
new file mode 100644
index 000000000..6b6e65fa7
--- /dev/null
+++ b/plugins/experimental/stek_share/example_server_conf.yaml
@@ -0,0 +1,16 @@
+server_id: 1
+address: 127.0.0.1
+port: 10001
+asio_thread_pool_size: 4
+heart_beat_interval: 100
+election_timeout_lower_bound: 200
+election_timeout_upper_bound: 400
+reserved_log_items: 5
+snapshot_distance: 5
+client_req_timeout: 3000 # this is in milliseconds
+key_update_interval: 3600 # this is in seconds
+server_list_file: /abs/path/to/server_list.yaml
+root_cert_file: /abs/path/to/ca.pem
+server_cert_file: /abs/path/to/server.pem
+server_key_file: /abs/path/to/server.key
+cert_verify_str: /C=US/ST=IL/O=Yahoo/OU=Edge/CN=localhost
diff --git a/plugins/experimental/stek_share/example_server_list.yaml b/plugins/experimental/stek_share/example_server_list.yaml
new file mode 100644
index 000000000..77d815f0a
--- /dev/null
+++ b/plugins/experimental/stek_share/example_server_list.yaml
@@ -0,0 +1,15 @@
+- server_id: 1
+ address: 127.0.0.1
+ port: 10001
+- server_id: 2
+ address: 127.0.0.1
+ port: 10002
+- server_id: 3
+ address: 127.0.0.1
+ port: 10003
+- server_id: 4
+ address: 127.0.0.1
+ port: 10004
+- server_id: 5
+ address: 127.0.0.1
+ port: 10005
diff --git a/plugins/experimental/stek_share/log_store.cc b/plugins/experimental/stek_share/log_store.cc
new file mode 100644
index 000000000..0bd31673a
--- /dev/null
+++ b/plugins/experimental/stek_share/log_store.cc
@@ -0,0 +1,263 @@
+/************************************************************************
+Copyright 2017-2019 eBay Inc.
+Author/Developer(s): Jung-Sang Ahn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+**************************************************************************/
+
+// This file is based on the example code from https://github.com/eBay/NuRaft/tree/master/examples
+
+#include <cassert>
+
+#include <libnuraft/nuraft.hxx>
+
+#include "log_store.h"
+
+STEKShareLogStore::STEKShareLogStore() : start_idx_(1)
+{
+ // Dummy entry for index 0.
+ nuraft::ptr<nuraft::buffer> buf = nuraft::buffer::alloc(sizeof(uint64_t));
+ logs_[0] = nuraft::cs_new<nuraft::log_entry>(0, buf);
+}
+
+STEKShareLogStore::~STEKShareLogStore() {}
+
+nuraft::ptr<nuraft::log_entry>
+STEKShareLogStore::make_clone(const nuraft::ptr<nuraft::log_entry> &entry)
+{
+ nuraft::ptr<nuraft::log_entry> clone =
+ nuraft::cs_new<nuraft::log_entry>(entry->get_term(), nuraft::buffer::clone(entry->get_buf()), entry->get_val_type());
+ return clone;
+}
+
+uint64_t
+STEKShareLogStore::next_slot() const
+{
+ std::lock_guard<std::mutex> l(logs_lock_);
+
+ // Exclude the dummy entry.
+ return start_idx_ + logs_.size() - 1;
+}
+
+uint64_t
+STEKShareLogStore::start_index() const
+{
+ return start_idx_;
+}
+
+nuraft::ptr<nuraft::log_entry>
+STEKShareLogStore::last_entry() const
+{
+ uint64_t next_idx = next_slot();
+ std::lock_guard<std::mutex> l(logs_lock_);
+ auto entry = logs_.find(next_idx - 1);
+ if (entry == logs_.end()) {
+ entry = logs_.find(0);
+ }
+
+ return make_clone(entry->second);
+}
+
+uint64_t
+STEKShareLogStore::append(nuraft::ptr<nuraft::log_entry> &entry)
+{
+ nuraft::ptr<nuraft::log_entry> clone = make_clone(entry);
+
+ std::lock_guard<std::mutex> l(logs_lock_);
+ size_t idx = start_idx_ + logs_.size() - 1;
+ logs_[idx] = clone;
+ return idx;
+}
+
+void
+STEKShareLogStore::write_at(uint64_t index, nuraft::ptr<nuraft::log_entry> &entry)
+{
+ nuraft::ptr<nuraft::log_entry> clone = make_clone(entry);
+
+ // Discard all logs equal to or greater than "index".
+ std::lock_guard<std::mutex> l(logs_lock_);
+ auto itr = logs_.lower_bound(index);
+ while (itr != logs_.end()) {
+ itr = logs_.erase(itr);
+ }
+ logs_[index] = clone;
+}
+
+nuraft::ptr<std::vector<nuraft::ptr<nuraft::log_entry>>>
+STEKShareLogStore::log_entries(uint64_t start, uint64_t end)
+{
+ nuraft::ptr<std::vector<nuraft::ptr<nuraft::log_entry>>> ret = nuraft::cs_new<std::vector<nuraft::ptr<nuraft::log_entry>>>();
+
+ ret->resize(end - start);
+ uint64_t cc = 0;
+ for (uint64_t i = start; i < end; ++i) {
+ nuraft::ptr<nuraft::log_entry> src = nullptr;
+ {
+ std::lock_guard<std::mutex> l(logs_lock_);
+ auto entry = logs_.find(i);
+ if (entry == logs_.end()) {
+ entry = logs_.find(0);
+ assert(0);
+ }
+ src = entry->second;
+ }
+ (*ret)[cc++] = make_clone(src);
+ }
+ return ret;
+}
+
+nuraft::ptr<std::vector<nuraft::ptr<nuraft::log_entry>>>
+STEKShareLogStore::log_entries_ext(uint64_t start, uint64_t end, int64_t batch_size_hint_in_bytes)
+{
+ nuraft::ptr<std::vector<nuraft::ptr<nuraft::log_entry>>> ret = nuraft::cs_new<std::vector<nuraft::ptr<nuraft::log_entry>>>();
+
+ if (batch_size_hint_in_bytes < 0) {
+ return ret;
+ }
+
+ size_t accum_size = 0;
+ for (uint64_t i = start; i < end; ++i) {
+ nuraft::ptr<nuraft::log_entry> src = nullptr;
+ {
+ std::lock_guard<std::mutex> l(logs_lock_);
+ auto entry = logs_.find(i);
+ if (entry == logs_.end()) {
+ entry = logs_.find(0);
+ assert(0);
+ }
+ src = entry->second;
+ }
+ ret->push_back(make_clone(src));
+ accum_size += src->get_buf().size();
+ if (batch_size_hint_in_bytes && accum_size >= (uint64_t)batch_size_hint_in_bytes) {
+ break;
+ }
+ }
+ return ret;
+}
+
+nuraft::ptr<nuraft::log_entry>
+STEKShareLogStore::entry_at(uint64_t index)
+{
+ nuraft::ptr<nuraft::log_entry> src = nullptr;
+ {
+ std::lock_guard<std::mutex> l(logs_lock_);
+ auto entry = logs_.find(index);
+ if (entry == logs_.end()) {
+ entry = logs_.find(0);
+ }
+ src = entry->second;
+ }
+ return make_clone(src);
+}
+
+uint64_t
+STEKShareLogStore::term_at(uint64_t index)
+{
+ uint64_t term = 0;
+ {
+ std::lock_guard<std::mutex> l(logs_lock_);
+ auto entry = logs_.find(index);
+ if (entry == logs_.end()) {
+ entry = logs_.find(0);
+ }
+ term = entry->second->get_term();
+ }
+ return term;
+}
+
+nuraft::ptr<nuraft::buffer>
+STEKShareLogStore::pack(uint64_t index, int32_t cnt)
+{
+ std::vector<nuraft::ptr<nuraft::buffer>> logs;
+
+ size_t size_total = 0;
+ for (uint64_t i = index; i < index + cnt; ++i) {
+ nuraft::ptr<nuraft::log_entry> le = nullptr;
+ {
+ std::lock_guard<std::mutex> l(logs_lock_);
+ le = logs_[i];
+ }
+ assert(le.get());
+ nuraft::ptr<nuraft::buffer> buf = le->serialize();
+ size_total += buf->size();
+ logs.push_back(buf);
+ }
+
+ nuraft::ptr<nuraft::buffer> buf_out = nuraft::buffer::alloc(sizeof(int32_t) + cnt * sizeof(int32_t) + size_total);
+ buf_out->pos(0);
+ buf_out->put((int32_t)cnt);
+
+ for (auto &entry : logs) {
+ nuraft::ptr<nuraft::buffer> &bb = entry;
+ buf_out->put((int32_t)bb->size());
+ buf_out->put(*bb);
+ }
+ return buf_out;
+}
+
+void
+STEKShareLogStore::apply_pack(uint64_t index, nuraft::buffer &pack)
+{
+ pack.pos(0);
+ int32_t num_logs = pack.get_int();
+
+ for (int32_t i = 0; i < num_logs; ++i) {
+ uint64_t cur_idx = index + i;
+ int32_t buf_size = pack.get_int();
+
+ nuraft::ptr<nuraft::buffer> buf_local = nuraft::buffer::alloc(buf_size);
+ pack.get(buf_local);
+
+ nuraft::ptr<nuraft::log_entry> le = nuraft::log_entry::deserialize(*buf_local);
+ {
+ std::lock_guard<std::mutex> l(logs_lock_);
+ logs_[cur_idx] = le;
+ }
+ }
+
+ {
+ std::lock_guard<std::mutex> l(logs_lock_);
+ auto entry = logs_.upper_bound(0);
+ if (entry != logs_.end()) {
+ start_idx_ = entry->first;
+ } else {
+ start_idx_ = 1;
+ }
+ }
+}
+
+bool
+STEKShareLogStore::compact(uint64_t last_log_index)
+{
+ std::lock_guard<std::mutex> l(logs_lock_);
+ for (uint64_t i = start_idx_; i <= last_log_index; ++i) {
+ auto entry = logs_.find(i);
+ if (entry != logs_.end()) {
+ logs_.erase(entry);
+ }
+ }
+
+ // WARNING:
+ // Even though nothing has been erased, we should set "start_idx_" to new index.
+ if (start_idx_ <= last_log_index) {
+ start_idx_ = last_log_index + 1;
+ }
+
+ return true;
+}
+
+void
+STEKShareLogStore::close()
+{
+}
diff --git a/plugins/experimental/stek_share/log_store.h b/plugins/experimental/stek_share/log_store.h
new file mode 100644
index 000000000..6f6659cde
--- /dev/null
+++ b/plugins/experimental/stek_share/log_store.h
@@ -0,0 +1,76 @@
+/************************************************************************
+Copyright 2017-2019 eBay Inc.
+Author/Developer(s): Jung-Sang Ahn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+**************************************************************************/
+
+// This file is based on the example code from https://github.com/eBay/NuRaft/tree/master/examples
+
+#pragma once
+
+#include <atomic>
+#include <map>
+#include <mutex>
+
+#include <libnuraft/log_store.hxx>
+
+class STEKShareLogStore : public nuraft::log_store
+{
+public:
+ STEKShareLogStore();
+
+ ~STEKShareLogStore();
+
+ __nocopy__(STEKShareLogStore);
+
+ uint64_t next_slot() const;
+
+ uint64_t start_index() const;
+
+ nuraft::ptr<nuraft::log_entry> last_entry() const;
+
+ uint64_t append(nuraft::ptr<nuraft::log_entry> &entry);
+
+ void write_at(uint64_t index, nuraft::ptr<nuraft::log_entry> &entry);
+
+ nuraft::ptr<std::vector<nuraft::ptr<nuraft::log_entry>>> log_entries(uint64_t start, uint64_t end);
+
+ nuraft::ptr<std::vector<nuraft::ptr<nuraft::log_entry>>> log_entries_ext(uint64_t start, uint64_t end,
+ int64_t batch_size_hint_in_bytes = 0);
+
+ nuraft::ptr<nuraft::log_entry> entry_at(uint64_t index);
+
+ uint64_t term_at(uint64_t index);
+
+ nuraft::ptr<nuraft::buffer> pack(uint64_t index, int32_t cnt);
+
+ void apply_pack(uint64_t index, nuraft::buffer &pack);
+
+ bool compact(uint64_t last_log_index);
+
+ bool
+ flush()
+ {
+ return true;
+ }
+
+ void close();
+
+private:
+ static nuraft::ptr<nuraft::log_entry> make_clone(const nuraft::ptr<nuraft::log_entry> &entry);
+
+ std::map<uint64_t, nuraft::ptr<nuraft::log_entry>> logs_;
+ mutable std::mutex logs_lock_;
+ std::atomic<uint64_t> start_idx_;
+};
diff --git a/plugins/experimental/stek_share/state_machine.h b/plugins/experimental/stek_share/state_machine.h
new file mode 100644
index 000000000..26a3995e8
--- /dev/null
+++ b/plugins/experimental/stek_share/state_machine.h
@@ -0,0 +1,237 @@
+/************************************************************************
+Copyright 2017-2019 eBay Inc.
+Author/Developer(s): Jung-Sang Ahn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+**************************************************************************/
+
+// This file is based on the example code from https://github.com/eBay/NuRaft/tree/master/examples
+
+#pragma once
+
+#include <atomic>
+#include <cassert>
+#include <iostream>
+#include <mutex>
+#include <cstring>
+
+#include <libnuraft/nuraft.hxx>
+
+#include "common.h"
+#include "stek_utils.h"
+
+class STEKShareSM : public nuraft::state_machine
+{
+public:
+ STEKShareSM() : last_committed_idx_(0) {}
+
+ ~STEKShareSM() {}
+
+ nuraft::ptr<nuraft::buffer>
+ pre_commit(const uint64_t log_idx, nuraft::buffer &data)
+ {
+ return nullptr;
+ }
+
+ nuraft::ptr<nuraft::buffer>
+ commit(const uint64_t log_idx, nuraft::buffer &data)
+ {
+ // Extract bytes from "data".
+ size_t len = 0;
+ nuraft::buffer_serializer bs_data(data);
+ void *byte_array = bs_data.get_bytes(len);
+ // TSDebug(PLUGIN, "commit %lu: %s", log_idx, hex_str(std::string(reinterpret_cast<char *>(byte_array), len)).c_str());
+
+ assert(len == SSL_TICKET_KEY_SIZE);
+
+ {
+ std::lock_guard<std::mutex> l(stek_lock_);
+ std::memcpy(&stek_, byte_array, len);
+ received_stek_ = true;
+ }
+
+ // Update last committed index number.
+ last_committed_idx_ = log_idx;
+
+ nuraft::ptr<nuraft::buffer> ret = nuraft::buffer::alloc(sizeof(log_idx));
+ nuraft::buffer_serializer bs_ret(ret);
+ bs_ret.put_u64(log_idx);
+
+ return ret;
+ }
+
+ bool
+ received_stek(ssl_ticket_key_t *curr_stek)
+ {
+ std::lock_guard<std::mutex> l(stek_lock_);
+ if (!received_stek_) {
+ return false;
+ }
+
+ received_stek_ = false;
+
+ if (std::memcmp(curr_stek, &stek_, SSL_TICKET_KEY_SIZE != 0)) {
+ std::memcpy(curr_stek, &stek_, SSL_TICKET_KEY_SIZE);
+ return true;
+ }
+
+ return false;
+ }
+
+ void
+ commit_config(const uint64_t log_idx, nuraft::ptr<nuraft::cluster_config> &new_conf)
+ {
+ // Nothing to do with configuration change. Just update committed index.
+ last_committed_idx_ = log_idx;
+ }
+
+ void
+ rollback(const uint64_t log_idx, nuraft::buffer &data)
+ {
+ // Nothing to do here since we don't have pre-commit.
+ }
+
+ int
+ read_logical_snp_obj(nuraft::snapshot &s, void *&user_snp_ctx, uint64_t obj_id, nuraft::ptr<nuraft::buffer> &data_out,
+ bool &is_last_obj)
+ {
+ // TSDebug(PLUGIN, "read snapshot %lu term %lu object ID %lu", s.get_last_log_idx(), s.get_last_log_term(), obj_id);
+
+ is_last_obj = true;
+
+ {
+ std::lock_guard<std::mutex> l(snapshot_lock_);
+ if (snapshot_ == nullptr || snapshot_->snapshot_->get_last_log_idx() != s.get_last_log_idx()) {
+ data_out = nullptr;
+ return -1;
+ } else {
+ data_out = nuraft::buffer::alloc(sizeof(int) + SSL_TICKET_KEY_SIZE);
+ nuraft::buffer_serializer bs(data_out);
+ bs.put_bytes(reinterpret_cast<const void *>(&snapshot_->stek_), SSL_TICKET_KEY_SIZE);
+ return 0;
+ }
+ }
+ }
+
+ void
+ save_logical_snp_obj(nuraft::snapshot &s, uint64_t &obj_id, nuraft::buffer &data, bool is_first_obj, bool is_last_obj)
+ {
+ // TSDebug(PLUGIN, "save snapshot %lu term %lu object ID %lu", s.get_last_log_idx(), s.get_last_log_term(), obj_id);
+
+ size_t len = 0;
+ nuraft::buffer_serializer bs_data(data);
+ void *byte_array = bs_data.get_bytes(len);
+
+ assert(len == SSL_TICKET_KEY_SIZE);
+
+ ssl_ticket_key_t local_stek;
+ std::memcpy(&local_stek, byte_array, len);
+
+ nuraft::ptr<nuraft::buffer> snp_buf = s.serialize();
+ nuraft::ptr<nuraft::snapshot> ss = nuraft::snapshot::deserialize(*snp_buf);
+ nuraft::ptr<struct snapshot_ctx> ctx = nuraft::cs_new<struct snapshot_ctx>(ss, local_stek);
+
+ {
+ std::lock_guard<std::mutex> l(snapshot_lock_);
+ snapshot_ = ctx;
+ }
+
+ obj_id++;
+ }
+
+ bool
+ apply_snapshot(nuraft::snapshot &s)
+ {
+ // TSDebug(PLUGIN, "apply snapshot %lu term %lu", s.get_last_log_idx(), s.get_last_log_term());
+
+ {
+ std::lock_guard<std::mutex> l(snapshot_lock_);
+ if (snapshot_ != nullptr) {
+ std::lock_guard<std::mutex> ll(stek_lock_);
+ std::memcpy(&stek_, &snapshot_->stek_, SSL_TICKET_KEY_SIZE);
+ received_stek_ = true;
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ void
+ free_user_snp_ctx(void *&user_snp_ctx)
+ {
+ }
+
+ nuraft::ptr<nuraft::snapshot>
+ last_snapshot()
+ {
+ // Just return the latest snapshot.
+ std::lock_guard<std::mutex> l(snapshot_lock_);
+ if (snapshot_ != nullptr) {
+ return snapshot_->snapshot_;
+ }
+ return nullptr;
+ }
+
+ uint64_t
+ last_commit_index()
+ {
+ return last_committed_idx_;
+ }
+
+ void
+ create_snapshot(nuraft::snapshot &s, nuraft::async_result<bool>::handler_type &when_done)
+ {
+ // TSDebug(PLUGIN, "create snapshot %lu term %lu", s.get_last_log_idx(), s.get_last_log_term());
+
+ ssl_ticket_key_t local_stek;
+ {
+ std::lock_guard<std::mutex> l(stek_lock_);
+ std::memcpy(&local_stek, &stek_, SSL_TICKET_KEY_SIZE);
+ }
+
+ nuraft::ptr<nuraft::buffer> snp_buf = s.serialize();
+ nuraft::ptr<nuraft::snapshot> ss = nuraft::snapshot::deserialize(*snp_buf);
+ nuraft::ptr<struct snapshot_ctx> ctx = nuraft::cs_new<struct snapshot_ctx>(ss, local_stek);
+
+ {
+ std::lock_guard<std::mutex> l(snapshot_lock_);
+ snapshot_ = ctx;
+ }
+
+ nuraft::ptr<std::exception> except(nullptr);
+ bool ret = true;
+ when_done(ret, except);
+ }
+
+private:
+ struct snapshot_ctx {
+ snapshot_ctx(nuraft::ptr<nuraft::snapshot> &s, ssl_ticket_key_t key) : snapshot_(s), stek_(key) {}
+ nuraft::ptr<nuraft::snapshot> snapshot_;
+ ssl_ticket_key_t stek_;
+ };
+
+ // Last committed Raft log number.
+ std::atomic<uint64_t> last_committed_idx_;
+
+ nuraft::ptr<struct snapshot_ctx> snapshot_;
+
+ // Mutex for snapshot.
+ std::mutex snapshot_lock_;
+
+ bool received_stek_ = false;
+
+ ssl_ticket_key_t stek_;
+
+ std::mutex stek_lock_;
+};
diff --git a/plugins/experimental/stek_share/state_manager.h b/plugins/experimental/stek_share/state_manager.h
new file mode 100644
index 000000000..63e16249e
--- /dev/null
+++ b/plugins/experimental/stek_share/state_manager.h
@@ -0,0 +1,102 @@
+/************************************************************************
+Copyright 2017-2019 eBay Inc.
+Author/Developer(s): Jung-Sang Ahn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+**************************************************************************/
+
+// This file is based on the example code from https://github.com/eBay/NuRaft/tree/master/examples
+
+#pragma once
+
+#include <libnuraft/nuraft.hxx>
+
+#include "log_store.h"
+
+class STEKShareSMGR : public nuraft::state_mgr
+{
+public:
+ STEKShareSMGR(int srv_id, const std::string &endpoint, std::map<int, std::string> server_list)
+ : my_id_(srv_id), my_endpoint_(endpoint), cur_log_store_(nuraft::cs_new<STEKShareLogStore>())
+ {
+ my_srv_config_ = nuraft::cs_new<nuraft::srv_config>(srv_id, endpoint);
+
+ // Initial cluster config, read from the list loaded from the configuration file.
+ saved_config_ = nuraft::cs_new<nuraft::cluster_config>();
+ for (auto const &s : server_list) {
+ int server_id = s.first;
+ std::string endpoint = s.second;
+ nuraft::ptr<nuraft::srv_config> new_server = nuraft::cs_new<nuraft::srv_config>(server_id, endpoint);
+ saved_config_->get_servers().push_back(new_server);
+ }
+ }
+
+ ~STEKShareSMGR() {}
+
+ nuraft::ptr<nuraft::cluster_config>
+ load_config()
+ {
+ return saved_config_;
+ }
+
+ void
+ save_config(const nuraft::cluster_config &config)
+ {
+ nuraft::ptr<nuraft::buffer> buf = config.serialize();
+ saved_config_ = nuraft::cluster_config::deserialize(*buf);
+ }
+
+ void
+ save_state(const nuraft::srv_state &state)
+ {
+ nuraft::ptr<nuraft::buffer> buf = state.serialize();
+ saved_state_ = nuraft::srv_state::deserialize(*buf);
+ }
+
+ nuraft::ptr<nuraft::srv_state>
+ read_state()
+ {
+ return saved_state_;
+ }
+
+ nuraft::ptr<nuraft::log_store>
+ load_log_store()
+ {
+ return cur_log_store_;
+ }
+
+ int32_t
+ server_id()
+ {
+ return my_id_;
+ }
+
+ void
+ system_exit(const int exit_code)
+ {
+ }
+
+ nuraft::ptr<nuraft::srv_config>
+ get_srv_config() const
+ {
+ return my_srv_config_;
+ }
+
+private:
+ int my_id_;
+ std::string my_endpoint_;
+ nuraft::ptr<STEKShareLogStore> cur_log_store_;
+ nuraft::ptr<nuraft::srv_config> my_srv_config_;
+ nuraft::ptr<nuraft::cluster_config> saved_config_;
+ nuraft::ptr<nuraft::srv_state> saved_state_;
+};
diff --git a/plugins/experimental/stek_share/stek_share.cc b/plugins/experimental/stek_share/stek_share.cc
new file mode 100644
index 000000000..810c4f75b
--- /dev/null
+++ b/plugins/experimental/stek_share/stek_share.cc
@@ -0,0 +1,445 @@
+/** @file
+
+ stek_share.cc
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ */
+
+#include <iostream>
+#include <fstream>
+#include <thread>
+#include <chrono>
+
+#include <openssl/ssl.h>
+#include <ts/ts.h>
+#include <ts/apidefs.h>
+#include <libnuraft/nuraft.hxx>
+#include <yaml-cpp/yaml.h>
+
+#include "state_machine.h"
+#include "state_manager.h"
+#include "stek_share.h"
+#include "stek_utils.h"
+#include "common.h"
+
+using raft_result = nuraft::cmd_result<nuraft::ptr<nuraft::buffer>>;
+
+PluginThreads plugin_threads;
+
+static STEKShareServer stek_share_server;
+static const nuraft::raft_params::return_method_type CALL_TYPE = nuraft::raft_params::blocking;
+// static const nuraft::raft_params::return_method_type CALL_TYPE = nuraft::raft_params::async_handler;
+
+static int
+shutdown_handler(TSCont contp, TSEvent event, void *edata)
+{
+ if (event == TS_EVENT_LIFECYCLE_SHUTDOWN) {
+ plugin_threads.terminate();
+ stek_share_server.launcher_.shutdown();
+ }
+ return 0;
+}
+
+bool
+cert_verification(const std::string &sn)
+{
+ if (sn.compare(stek_share_server.cert_verify_str_) != 0) {
+ TSDebug(PLUGIN, "Cert incorrect, expecting: %s, got: %s", stek_share_server.cert_verify_str_.c_str(), sn.c_str());
+ return false;
+ }
+ return true;
+}
+
+int
+init_raft(nuraft::ptr<nuraft::state_machine> sm_instance)
+{
+ // State machine.
+ stek_share_server.smgr_ =
+ nuraft::cs_new<STEKShareSMGR>(stek_share_server.server_id_, stek_share_server.endpoint_, stek_share_server.server_list_);
+
+ // State manager.
+ stek_share_server.sm_ = sm_instance;
+
+ // ASIO options.
+ nuraft::asio_service::options asio_opts;
+ asio_opts.thread_pool_size_ = stek_share_server.asio_thread_pool_size_;
+ asio_opts.enable_ssl_ = true;
+ asio_opts.verify_sn_ = cert_verification;
+ asio_opts.root_cert_file_ = stek_share_server.root_cert_file_;
+ asio_opts.server_cert_file_ = stek_share_server.server_cert_file_;
+ asio_opts.server_key_file_ = stek_share_server.server_key_file_;
+
+ // Raft parameters.
+ nuraft::raft_params params;
+ params.heart_beat_interval_ = stek_share_server.heart_beat_interval_;
+ params.election_timeout_lower_bound_ = stek_share_server.election_timeout_lower_bound_;
+ params.election_timeout_upper_bound_ = stek_share_server.election_timeout_upper_bound_;
+ params.reserved_log_items_ = stek_share_server.reserved_log_items_;
+ params.snapshot_distance_ = stek_share_server.snapshot_distance_;
+ params.client_req_timeout_ = stek_share_server.client_req_timeout_;
+
+ // According to this method, "append_log" function should be handled differently.
+ params.return_method_ = CALL_TYPE;
+
+ // Initialize Raft server.
+ stek_share_server.raft_instance_ = stek_share_server.launcher_.init(stek_share_server.sm_, stek_share_server.smgr_, nullptr,
+ stek_share_server.port_, asio_opts, params);
+
+ if (!stek_share_server.raft_instance_) {
+ TSDebug(PLUGIN, "Failed to initialize launcher.");
+ return -1;
+ }
+
+ TSDebug(PLUGIN, "Raft instance initialization done.");
+ return 0;
+}
+
+int
+set_server_info(int argc, const char *argv[])
+{
+ // Get server ID.
+ YAML::Node server_conf;
+ try {
+ server_conf = YAML::LoadFile(argv[1]);
+ } catch (YAML::BadFile &e) {
+ TSEmergency("[%s] Cannot load configuration file: %s.", PLUGIN, e.what());
+ } catch (std::exception &e) {
+ TSEmergency("[%s] Unknown error while loading configuration file: %s.", PLUGIN, e.what());
+ }
+
+ if (server_conf["server_id"]) {
+ stek_share_server.server_id_ = server_conf["server_id"].as<int>();
+ if (stek_share_server.server_id_ < 1) {
+ TSDebug(PLUGIN, "Wrong server id (must be >= 1): %d", stek_share_server.server_id_);
+ return -1;
+ }
+ } else {
+ TSDebug(PLUGIN, "Must specify server id in the configuration file.");
+ return -1;
+ }
+
+ // Get server address and port.
+ if (server_conf["address"]) {
+ stek_share_server.addr_ = server_conf["address"].as<std::string>();
+ } else {
+ TSDebug(PLUGIN, "Must specify server address in the configuration file.");
+ return -1;
+ }
+
+ if (server_conf["port"]) {
+ stek_share_server.port_ = server_conf["port"].as<int>();
+ } else {
+ TSDebug(PLUGIN, "Must specify server port in the configuration file.");
+ return -1;
+ }
+
+ stek_share_server.endpoint_ = stek_share_server.addr_ + ":" + std::to_string(stek_share_server.port_);
+
+ if (server_conf["asio_thread_pool_size"]) {
+ stek_share_server.asio_thread_pool_size_ = server_conf["asio_thread_pool_size"].as<size_t>();
+ }
+
+ if (server_conf["heart_beat_interval"]) {
+ stek_share_server.heart_beat_interval_ = server_conf["heart_beat_interval"].as<int>();
+ }
+
+ if (server_conf["election_timeout_lower_bound"]) {
+ stek_share_server.election_timeout_lower_bound_ = server_conf["election_timeout_lower_bound"].as<int>();
+ }
+
+ if (server_conf["election_timeout_upper_bound"]) {
+ stek_share_server.election_timeout_upper_bound_ = server_conf["election_timeout_upper_bound"].as<int>();
+ }
+
+ if (server_conf["reserved_log_items"]) {
+ stek_share_server.reserved_log_items_ = server_conf["reserved_log_items"].as<int>();
+ }
+
+ if (server_conf["snapshot_distance"]) {
+ stek_share_server.snapshot_distance_ = server_conf["snapshot_distance"].as<int>();
+ }
+
+ if (server_conf["client_req_timeout"]) {
+ stek_share_server.client_req_timeout_ = server_conf["client_req_timeout"].as<int>();
+ }
+
+ if (server_conf["key_update_interval"]) {
+ stek_share_server.key_update_interval_ = server_conf["key_update_interval"].as<int>();
+ } else {
+ TSDebug(PLUGIN, "Must specify server key update interval in the configuration file.");
+ return -1;
+ }
+
+ if (server_conf["server_list_file"]) {
+ YAML::Node server_list;
+ try {
+ server_list = YAML::LoadFile(server_conf["server_list_file"].as<std::string>());
+ } catch (YAML::BadFile &e) {
+ TSEmergency("[%s] Cannot load server list file: %s.", PLUGIN, e.what());
+ } catch (std::exception &e) {
+ TSEmergency("[%s] Unknown error while loading server list file: %s.", PLUGIN, e.what());
+ }
+
+ std::string cluster_list_str = "";
+ cluster_list_str += "\nSTEK Share Cluster Server List:";
+ for (auto it = server_list.begin(); it != server_list.end(); ++it) {
+ YAML::Node server_info = it->as<YAML::Node>();
+ if (server_info["server_id"] && server_info["address"] && server_info["port"]) {
+ int server_id = server_info["server_id"].as<int>();
+ std::string address = server_info["address"].as<std::string>();
+ int port = server_info["port"].as<int>();
+ std::string endpoint = address + ":" + std::to_string(port);
+ stek_share_server.server_list_[server_id] = endpoint;
+ cluster_list_str += "\n " + std::to_string(server_id) + ", " + endpoint;
+ } else {
+ TSDebug(PLUGIN, "Wrong server list format.");
+ return -1;
+ }
+ }
+ TSDebug(PLUGIN, "%s", cluster_list_str.c_str());
+ } else {
+ TSDebug(PLUGIN, "Must specify server list file in the configuration file.");
+ return -1;
+ }
+
+ // TODO: check cert and key files exist
+ if (server_conf["root_cert_file"]) {
+ stek_share_server.root_cert_file_ = server_conf["root_cert_file"].as<std::string>();
+ } else {
+ TSDebug(PLUGIN, "Must specify root ca file in the configuration file.");
+ return -1;
+ }
+
+ if (server_conf["server_cert_file"]) {
+ stek_share_server.server_cert_file_ = server_conf["server_cert_file"].as<std::string>();
+ } else {
+ TSDebug(PLUGIN, "Must specify server cert file in the configuration file.");
+ return -1;
+ }
+
+ if (server_conf["server_key_file"]) {
+ stek_share_server.server_key_file_ = server_conf["server_key_file"].as<std::string>();
+ } else {
+ TSDebug(PLUGIN, "Must specify server key file in the configuration file.");
+ return -1;
+ }
+
+ if (server_conf["cert_verify_str"]) {
+ stek_share_server.cert_verify_str_ = server_conf["cert_verify_str"].as<std::string>();
+ } else {
+ TSDebug(PLUGIN, "Must specify cert verify string in the configuration file.");
+ return -1;
+ }
+
+ return 0;
+}
+
+void
+handle_result(raft_result &result, nuraft::ptr<std::exception> &err)
+{
+ if (result.get_result_code() != nuraft::cmd_result_code::OK) {
+ // Something went wrong.
+ // This means committing this log failed, but the log itself is still in the log store.
+ TSDebug(PLUGIN, "Replication failed: %d", result.get_result_code());
+ return;
+ }
+ TSDebug(PLUGIN, "Replication succeeded.");
+}
+
+void
+append_log(const void *data, int data_len)
+{
+ // Create a new log which will contain 4-byte length and string data.
+ nuraft::ptr<nuraft::buffer> new_log = nuraft::buffer::alloc(sizeof(int) + data_len);
+ nuraft::buffer_serializer bs(new_log);
+ bs.put_bytes(data, data_len);
+
+ // Do append.
+ nuraft::ptr<raft_result> ret = stek_share_server.raft_instance_->append_entries({new_log});
+
+ if (!ret->get_accepted()) {
+ // Log append rejected, usually because this node is not a leader.
+ TSDebug(PLUGIN, "Replication failed: %d", ret->get_result_code());
+ return;
+ }
+
+ // Log append accepted, but that doesn't mean the log is committed.
+ // Commit result can be obtained below.
+ if (CALL_TYPE == nuraft::raft_params::blocking) {
+ // Blocking mode:
+ // "append_entries" returns after getting a consensus, so that "ret" already has the result from state machine.
+ nuraft::ptr<std::exception> err(nullptr);
+ handle_result(*ret, err);
+ } else if (CALL_TYPE == nuraft::raft_params::async_handler) {
+ // Async mode:
+ // "append_entries" returns immediately. "handle_result" will be invoked asynchronously, after getting a consensus.
+ ret->when_ready(std::bind(handle_result, std::placeholders::_1, std::placeholders::_2));
+ } else {
+ assert(0);
+ }
+}
+
+void
+print_status()
+{
+ // For debugging
+ nuraft::ptr<nuraft::log_store> ls = stek_share_server.smgr_->load_log_store();
+ std::string status_str = "";
+ status_str += "\n Server ID: " + std::to_string(stek_share_server.server_id_);
+ status_str += "\n Leader ID: " + std::to_string(stek_share_server.raft_instance_->get_leader());
+ status_str += "\n Raft log range: " + std::to_string(ls->start_index()) + " - " + std::to_string((ls->next_slot() - 1));
+ status_str += "\n Last committed index: " + std::to_string(stek_share_server.raft_instance_->get_committed_log_idx());
+ TSDebug(PLUGIN, "%s", status_str.c_str());
+}
+
+static void *
+stek_updater(void *arg)
+{
+ plugin_threads.store(::pthread_self());
+ ::pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, nullptr);
+ ::pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, nullptr);
+
+ ssl_ticket_key_t curr_stek;
+ time_t init_key_time = 0;
+
+ // Initial key to use before syncing up.
+ TSDebug(PLUGIN, "Generating initial STEK...");
+ if (generate_new_stek(&curr_stek, 0 /* fast start */) == 0) {
+ TSDebug(PLUGIN, "Generate initial STEK succeeded: %s",
+ hex_str(std::string(reinterpret_cast<char *>(&curr_stek), SSL_TICKET_KEY_SIZE)).c_str());
+
+ std::memcpy(&stek_share_server.ticket_keys_[0], &curr_stek, SSL_TICKET_KEY_SIZE);
+
+ TSDebug(PLUGIN, "Updating SSL Ticket Key...");
+ if (TSSslTicketKeyUpdate(reinterpret_cast<char *>(stek_share_server.ticket_keys_), SSL_TICKET_KEY_SIZE) == TS_ERROR) {
+ TSDebug(PLUGIN, "Update SSL Ticket Key failed.");
+ } else {
+ TSDebug(PLUGIN, "Update SSL Ticket Key succeeded.");
+ init_key_time = time(nullptr);
+ }
+ } else {
+ TSFatal("Generate initial STEK failed.");
+ }
+
+ // Since we're using a pre-configured cluster, we need to have >= 3 nodes in the clust
+ // to initialize. Busy check before that.
+ while (!plugin_threads.is_shut_down()) {
+ if (!stek_share_server.raft_instance_->is_initialized()) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(250));
+ continue;
+ }
+
+ if (stek_share_server.raft_instance_->is_leader()) {
+ // We only need to generate new STEK if this server is the leader.
+ // Otherwise we wake up every 10 seconds to see whether a new STEK has been received.
+ if (init_key_time != 0 && time(nullptr) - init_key_time < stek_share_server.key_update_interval_) {
+ // If we got here after starting up, that means the initial key is still valid and we can send it to everyone else.
+ stek_share_server.last_updated_ = init_key_time;
+ TSDebug(PLUGIN, "Using initial STEK: %s",
+ hex_str(std::string(reinterpret_cast<char *>(&curr_stek), SSL_TICKET_KEY_SIZE)).c_str());
+ append_log(reinterpret_cast<const void *>(&curr_stek), SSL_TICKET_KEY_SIZE);
+
+ } else if (time(nullptr) - stek_share_server.last_updated_ >= stek_share_server.key_update_interval_) {
+ // Generate a new key as the last one has expired.
+ // Move the old key from ticket_keys_[0] to ticket_keys_[1], then put the new key in ticket_keys_[0].
+ TSDebug(PLUGIN, "Generating new STEK...");
+ if (generate_new_stek(&curr_stek, 1) == 0) {
+ TSDebug(PLUGIN, "Generate new STEK succeeded: %s",
+ hex_str(std::string(reinterpret_cast<char *>(&curr_stek), SSL_TICKET_KEY_SIZE)).c_str());
+
+ std::memcpy(&stek_share_server.ticket_keys_[1], &stek_share_server.ticket_keys_[0], SSL_TICKET_KEY_SIZE);
+ std::memcpy(&stek_share_server.ticket_keys_[0], &curr_stek, SSL_TICKET_KEY_SIZE);
+
+ TSDebug(PLUGIN, "Updating SSL Ticket Key...");
+ if (TSSslTicketKeyUpdate(reinterpret_cast<char *>(stek_share_server.ticket_keys_), SSL_TICKET_KEY_SIZE * 2) == TS_ERROR) {
+ TSDebug(PLUGIN, "Update SSL Ticket Key failed.");
+ } else {
+ stek_share_server.last_updated_ = time(nullptr);
+ TSDebug(PLUGIN, "Update SSL Ticket Key succeeded.");
+ TSDebug(PLUGIN, "Using new STEK: %s",
+ hex_str(std::string(reinterpret_cast<char *>(&curr_stek), SSL_TICKET_KEY_SIZE)).c_str());
+ append_log(reinterpret_cast<const void *>(&curr_stek), SSL_TICKET_KEY_SIZE);
+ }
+ } else {
+ TSFatal("Generate new STEK failed.");
+ }
+ }
+ init_key_time = 0;
+
+ } else {
+ init_key_time = 0;
+ auto sm = dynamic_cast<STEKShareSM *>(stek_share_server.sm_.get());
+
+ // Check whether we received a new key.
+ // TODO: retry updating STEK when failed
+ if (sm->received_stek(&curr_stek)) {
+ TSDebug(PLUGIN, "Received new STEK: %s",
+ hex_str(std::string(reinterpret_cast<char *>(&curr_stek), SSL_TICKET_KEY_SIZE)).c_str());
+
+ // Move the old key from ticket_keys_[0] to ticket_keys_[1], then put the new key in ticket_keys_[0].
+ std::memcpy(&stek_share_server.ticket_keys_[1], &stek_share_server.ticket_keys_[0], SSL_TICKET_KEY_SIZE);
+ std::memcpy(&stek_share_server.ticket_keys_[0], &curr_stek, SSL_TICKET_KEY_SIZE);
+
+ TSDebug(PLUGIN, "Updating SSL Ticket Key...");
+ if (TSSslTicketKeyUpdate(reinterpret_cast<char *>(stek_share_server.ticket_keys_), SSL_TICKET_KEY_SIZE * 2) == TS_ERROR) {
+ TSDebug(PLUGIN, "Update SSL Ticket Key failed.");
+ } else {
+ stek_share_server.last_updated_ = time(nullptr);
+ TSDebug(PLUGIN, "Update SSL Ticket Key succeeded.");
+ }
+ }
+ }
+
+ // Wakeup every 10 seconds to check whether there is a new key to use.
+ // We do this because if a server is lagging behind, either by losing connection or joining late,
+ // that server might receive multiple keys (the ones it missed) when it reconnects. Since we only need the
+ // most recent one, and to save time, we check back every 10 seconds in hope that the barrage of incoming
+ // keys has finished, and if not the second time around it'll definitely has.
+ std::this_thread::sleep_for(std::chrono::seconds(10));
+ }
+
+ return nullptr;
+}
+
+void
+TSPluginInit(int argc, const char *argv[])
+{
+ TSPluginRegistrationInfo info;
+
+ info.plugin_name = (char *)("stek_share");
+ info.vendor_name = (char *)("ats");
+ info.support_email = (char *)("ats-devel@yahooinc.com");
+
+ TSLifecycleHookAdd(TS_LIFECYCLE_SHUTDOWN_HOOK, TSContCreate(shutdown_handler, nullptr));
+
+ if (TSPluginRegister(&info) != TS_SUCCESS) {
+ TSError("Plugin registration failed.");
+ return;
+ }
+
+ if (argc < 2) {
+ TSError("Must specify config file.");
+ } else if (set_server_info(argc, argv) == 0 && init_raft(nuraft::cs_new<STEKShareSM>()) == 0) {
+ TSDebug(PLUGIN, "Server ID: %d, Endpoint: %s", stek_share_server.server_id_, stek_share_server.endpoint_.c_str());
+ TSThreadCreate(stek_updater, nullptr);
+ } else {
+ TSError("Raft initialization failed.");
+ }
+}
diff --git a/plugins/experimental/stek_share/stek_share.h b/plugins/experimental/stek_share/stek_share.h
new file mode 100644
index 000000000..14a8579bb
--- /dev/null
+++ b/plugins/experimental/stek_share/stek_share.h
@@ -0,0 +1,123 @@
+/************************************************************************
+Copyright 2017-2019 eBay Inc.
+Author/Developer(s): Jung-Sang Ahn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+**************************************************************************/
+
+// This file is based on the example code from https://github.com/eBay/NuRaft/tree/master/examples
+
+#pragma once
+
+#include <deque>
+#include <mutex>
+#include <time.h>
+
+#include <libnuraft/nuraft.hxx>
+
+#include "stek_utils.h"
+
+class STEKShareServer
+{
+public:
+ STEKShareServer() : server_id_(1), addr_("localhost"), port_(25000), sm_(nullptr), smgr_(nullptr), raft_instance_(nullptr)
+ {
+ last_updated_ = 0;
+ current_log_idx_ = 0;
+
+ // Default ASIO thread pool size: 4.
+ asio_thread_pool_size_ = 4;
+
+ // Default heart beat interval: 100 ms.
+ heart_beat_interval_ = 100;
+
+ // Default election timeout: 200~400 ms.
+ election_timeout_lower_bound_ = 200;
+ election_timeout_upper_bound_ = 400;
+
+ // Up to 5 logs will be preserved ahead the last snapshot.
+ reserved_log_items_ = 5;
+
+ // Snapshot will be created for every 5 log appends.
+ snapshot_distance_ = 5;
+
+ // Client timeout: 3000 ms.
+ client_req_timeout_ = 3000;
+
+ std::memset(ticket_keys_, 0, SSL_TICKET_KEY_SIZE * 2);
+ }
+
+ void
+ reset()
+ {
+ sm_.reset();
+ smgr_.reset();
+ raft_instance_.reset();
+ }
+
+ // Server ID.
+ int server_id_;
+
+ // Server address.
+ std::string addr_;
+
+ // Server port.
+ int port_;
+
+ // Endpoint: "<addr>:<port>".
+ std::string endpoint_;
+
+ // State machine.
+ nuraft::ptr<nuraft::state_machine> sm_;
+
+ // State manager.
+ nuraft::ptr<nuraft::state_mgr> smgr_;
+
+ // Raft launcher.
+ nuraft::raft_launcher launcher_;
+
+ // Raft server instance.
+ nuraft::ptr<nuraft::raft_server> raft_instance_;
+
+ // List of servers to auto add.
+ std::map<int, std::string> server_list_;
+
+ // STEK update interval.
+ int key_update_interval_;
+
+ // When was STEK last updated.
+ time_t last_updated_;
+
+ uint64_t current_log_idx_;
+
+ size_t asio_thread_pool_size_;
+
+ int heart_beat_interval_;
+
+ int election_timeout_lower_bound_;
+ int election_timeout_upper_bound_;
+
+ int reserved_log_items_;
+
+ int snapshot_distance_;
+
+ int client_req_timeout_;
+
+ // TLS related stuff.
+ std::string root_cert_file_;
+ std::string server_cert_file_;
+ std::string server_key_file_;
+ std::string cert_verify_str_;
+
+ ssl_ticket_key_t ticket_keys_[2];
+};
diff --git a/plugins/experimental/stek_share/stek_utils.cc b/plugins/experimental/stek_share/stek_utils.cc
new file mode 100644
index 000000000..89a3884b7
--- /dev/null
+++ b/plugins/experimental/stek_share/stek_utils.cc
@@ -0,0 +1,82 @@
+/** @file
+
+ stek_utils.cc - Deal with STEK
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ */
+
+#include <iostream>
+#include <cstring>
+#include <mutex>
+#include <cassert>
+#include <unistd.h>
+#include <sys/time.h>
+
+#include <openssl/rand.h>
+#include <ts/ts.h>
+#include <ts/apidefs.h>
+
+#include "stek_utils.h"
+#include "common.h"
+
+int
+get_good_random(char *buffer, int size, int need_good_entropy)
+{
+ FILE *fp;
+ int numread = 0;
+ char *rand_file_name;
+
+ /* /dev/random blocks until good entropy and can take up to 2 seconds per byte on idle machines */
+ /* /dev/urandom does not have entropy check, and is very quick.
+ * Caller decides quality needed */
+ rand_file_name = const_cast<char *>((need_good_entropy) ? /* Good & slow */ "/dev/random" : /*Fast*/ "/dev/urandom");
+
+ if (nullptr == (fp = fopen(rand_file_name, "r"))) {
+ return -1; /* failure */
+ }
+ numread = static_cast<int>(fread(buffer, 1, size, fp));
+ fclose(fp);
+
+ return ((numread == size) ? 0 /* success*/ : -1 /*failure*/);
+}
+
+int
+generate_new_stek(ssl_ticket_key_t *return_stek, int entropy_ensured)
+{
+ /* Generate a new Session-Ticket-Encryption-Key and places it into buffer
+ * provided return -1 on failure, 0 on success.
+ * if boolean global_key is set (inidcating it's the global key space),
+ will get global key lock before setting */
+
+ ssl_ticket_key_t new_key; // tmp local buffer
+
+ /* We create key in local buff to minimize lock time on global,
+ * because entropy ensuring can take a very long time e.g. 2 seconds per byte of entropy*/
+ if ((get_good_random(reinterpret_cast<char *>(&(new_key.aes_key)), SSL_KEY_LEN, (entropy_ensured) ? 1 : 0) != 0) ||
+ (get_good_random(reinterpret_cast<char *>(&(new_key.hmac_secret)), SSL_KEY_LEN, (entropy_ensured) ? 1 : 0) != 0) ||
+ (get_good_random(reinterpret_cast<char *>(&(new_key.key_name)), SSL_KEY_LEN, 0) != 0)) {
+ return -1; /* couldn't generate new STEK */
+ }
+
+ std::memcpy(return_stek, &new_key, SSL_TICKET_KEY_SIZE);
+ std::memset(&new_key, 0, SSL_TICKET_KEY_SIZE); // keep our stack clean
+
+ return 0; /* success */
+}
diff --git a/plugins/experimental/stek_share/stek_utils.h b/plugins/experimental/stek_share/stek_utils.h
new file mode 100644
index 000000000..c51a73bbb
--- /dev/null
+++ b/plugins/experimental/stek_share/stek_utils.h
@@ -0,0 +1,43 @@
+/** @file
+
+ stek_utils.h
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ */
+
+#pragma once
+
+/* STEK - Session Ticket Encryption Key stuff */
+#define STEK_MAX_LIFETIME 86400 // 24 hours max - should rotate STEK
+#define STEK_NOT_CHANGED_WARNING_INTERVAL (2 * STEK_MAX_LIFETIME) // warn on non-stek rotate every X secs.
+#define STEK_MAX_ENC_SIZE 512
+
+#define SSL_KEY_LEN 16
+
+typedef struct ssl_ticket_key // an STEK
+{
+ unsigned char key_name[SSL_KEY_LEN]; // tickets use this name to identify who encrypted
+ unsigned char hmac_secret[SSL_KEY_LEN];
+ unsigned char aes_key[SSL_KEY_LEN];
+} ssl_ticket_key_t;
+
+#define SSL_TICKET_KEY_SIZE sizeof(ssl_ticket_key_t)
+
+int generate_new_stek(ssl_ticket_key_t *return_stek, int entropy_ensured);
diff --git a/tests/gold_tests/pluginTest/stek_share/server_list.yaml b/tests/gold_tests/pluginTest/stek_share/server_list.yaml
new file mode 100644
index 000000000..77d815f0a
--- /dev/null
+++ b/tests/gold_tests/pluginTest/stek_share/server_list.yaml
@@ -0,0 +1,15 @@
+- server_id: 1
+ address: 127.0.0.1
+ port: 10001
+- server_id: 2
+ address: 127.0.0.1
+ port: 10002
+- server_id: 3
+ address: 127.0.0.1
+ port: 10003
+- server_id: 4
+ address: 127.0.0.1
+ port: 10004
+- server_id: 5
+ address: 127.0.0.1
+ port: 10005
diff --git a/tests/gold_tests/pluginTest/stek_share/ssl/self_signed.crt b/tests/gold_tests/pluginTest/stek_share/ssl/self_signed.crt
new file mode 100644
index 000000000..e90438c77
--- /dev/null
+++ b/tests/gold_tests/pluginTest/stek_share/ssl/self_signed.crt
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFfTCCA2WgAwIBAgIUEH/jBPLg2k3sO2+PKgnU+Rht2y4wDQYJKoZIhvcNAQEL
+BQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAklMMQ4wDAYDVQQKDAVZYWhvbzEN
+MAsGA1UECwwERWRnZTETMBEGA1UEAwwKc3Rlay1zaGFyZTAeFw0yMjAyMTAxNjIy
+MTdaFw0zMjAyMDgxNjIyMTdaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJJTDEO
+MAwGA1UECgwFWWFob28xDTALBgNVBAsMBEVkZ2UxEzARBgNVBAMMCnN0ZWstc2hh
+cmUwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDOJLcQ0P6yNQoYANlU
+231rpXK7tgiL0QXOv3vw7AosShkZj7NRPEIFRSFrzyAqJLfmeYBz0JPVjOraJ3CV
+8D9/2BUNdrbPGTgESEiDzyB4D3DSi8RHy11dwRozwi1YzDzcNoSii3xmE70bt0sT
+tzQ1L2lns+B1NoNBNP0F9VhYf0tvNsy5rRtgxFjZDpSiLXeqkf3IYEs0pEK+uk6+
+eXQ0uSTiAkfkQabw65NoFXaN9OxLyWDu4c+lxq7LKPRQ9O9lI05YaScA3d79ob/k
+qb0v4khS7cuYi19h7D5b0QZo1T/EA0HykMYvxfHHH3HviqQItxBYPnVxbwo2D6mF
+WS3Uupoh57QkR2HkX3MReZcHwgBDd7IciAyKB6a0VoKqlyLN82Mod4svHo8LGX1d
+zWWeOV89pa3txZ5gzilpMtoCwgGVaKn0u1mqEBx4azY8bJkm7OtSYC8GXtDbB7fo
+tptcXn3JWGgvtUAmZlaaSzlNocjORmjUIXr85kiaWNLmJpHNS5w3Ncf9Y7swxfwn
+BQmNiU0TdXsMHDDlGXoy+0Osw+GJmQijOIF2pQGnKsSozVtBIP4/ESKgAxh8+5Q1
+d0D0yG0OC/pcnam9Yx/qFRwvSUZ5X+hSam8ljwT1NzC74ATMvt9DU6AFSGK56Qqe
+/Y/QQShsUg1uIqa9L2hvamIOfwIDAQABo1MwUTAdBgNVHQ4EFgQU0G+dbyjOmCbn
+ay+x1u/lM9OY/fAwHwYDVR0jBBgwFoAU0G+dbyjOmCbnay+x1u/lM9OY/fAwDwYD
+VR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAq9zQnurNYUCsDjRXNU+h
+72i2zZCnWu5oKe6lCGn9hW3nGmRFAjfKb+fE3sNJxikRF7Cas+3mx1g7Z6riHjq9
+WMPKfg6H+ONXaxlfBbHCVrAkrflU1EVyPCXHVkapq4+h1HQAfEVpA13CXFnztTXu
+/tldQf2qDsloRdh9CNp/rNAzee2ouzy7SaDdTeA5DgNWhLwUQ4qCj+llD8me10lu
+YqyBWWdRG28Tt7GyeK3mJH9vwHsIA/hadOiDJ6ReS5Rm+MyxoRQY8QnnNS3h5UjZ
+wR0o3DrweyX0gWf4CI3Ycj5+gZvjI9kPd3deqCAh5YUj9yEYLZI1++Aud0zUSjPK
+R/Z9fLyjmADN0FV71aP3g6e/A9doyQYpNVg+spxCQnTCDGRuRhJv8Ubvehy9bCuw
+d+iMZXybqii8xsEdTZp/2L8ZarAe888gC/JEQw3JKo/NTaW0P6FS7LYAMLmya2ez
+5I4rT9BTTdwoxKSVbEro3f36U4hChS4YMYIsZ1sLjNljAQiUAAJeoJnVS/vCDw44
+YAQtgAIaIeMLqFgikf0U688r+78lZYIxhAuB2hWSUt7JDHGEq6+GZqh/3GLmSZmi
+Pqk/jV/esxEBBmxyP6hiwDKanfEQfpk6n5kxyftIRLGSg5EMtD00AA3Ono9uyr7e
+znK//SeO390xcNVQsi+8Ppw=
+-----END CERTIFICATE-----
diff --git a/tests/gold_tests/pluginTest/stek_share/ssl/self_signed.key b/tests/gold_tests/pluginTest/stek_share/ssl/self_signed.key
new file mode 100644
index 000000000..bbd6bbb74
--- /dev/null
+++ b/tests/gold_tests/pluginTest/stek_share/ssl/self_signed.key
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDOJLcQ0P6yNQoY
+ANlU231rpXK7tgiL0QXOv3vw7AosShkZj7NRPEIFRSFrzyAqJLfmeYBz0JPVjOra
+J3CV8D9/2BUNdrbPGTgESEiDzyB4D3DSi8RHy11dwRozwi1YzDzcNoSii3xmE70b
+t0sTtzQ1L2lns+B1NoNBNP0F9VhYf0tvNsy5rRtgxFjZDpSiLXeqkf3IYEs0pEK+
+uk6+eXQ0uSTiAkfkQabw65NoFXaN9OxLyWDu4c+lxq7LKPRQ9O9lI05YaScA3d79
+ob/kqb0v4khS7cuYi19h7D5b0QZo1T/EA0HykMYvxfHHH3HviqQItxBYPnVxbwo2
+D6mFWS3Uupoh57QkR2HkX3MReZcHwgBDd7IciAyKB6a0VoKqlyLN82Mod4svHo8L
+GX1dzWWeOV89pa3txZ5gzilpMtoCwgGVaKn0u1mqEBx4azY8bJkm7OtSYC8GXtDb
+B7fotptcXn3JWGgvtUAmZlaaSzlNocjORmjUIXr85kiaWNLmJpHNS5w3Ncf9Y7sw
+xfwnBQmNiU0TdXsMHDDlGXoy+0Osw+GJmQijOIF2pQGnKsSozVtBIP4/ESKgAxh8
++5Q1d0D0yG0OC/pcnam9Yx/qFRwvSUZ5X+hSam8ljwT1NzC74ATMvt9DU6AFSGK5
+6Qqe/Y/QQShsUg1uIqa9L2hvamIOfwIDAQABAoICAD5j2FAroNpYuSxYnW5UW9pH
+obj0OBPw+DwBskZReia7amtVFaWBgk3MBXh2oLqAkHQd0+W5e/THCJFsHGQU6XMM
++BoyEtQNQunw4pmaB66upavjh01fXGuytPZzT3wvnD/d9Dip1MWkNbj8ualG6nMq
+XVF4nHd9Py5uFiJGhi2KoU8Qm9eab83Svz06b3vCHRSvyMprcneFO3o0Mv7tDWGj
+o2kP3ahUwmzqL5vx2wbN2PJ7CW5jQ5Bd2Ks+Qut5pjbK/7w8XwShIgtLeCOBx/OF
+HfSTaepKTFz7vkfVtIXn/LubbMs4S/NLioiEmNwx7sGAfl7m0G67d7Cy/tCQFBFi
+JsdeZ0ekLH+nH+9QJzSFQzxDUrl5oGxuvtUa8HYbRUtkfaTzrHX/W3EO0Cgscs9y
+Pz6ARWunyanmttmWQEXorstjiRxOXP+QX+B3Uwjl18tDHs6dLUgpPv7Kl5nJ63KF
+B30ITVOK6e3sLi9y9AGVT8EVNQ7Zd7XLtXJ9lPOKjDec1TsSFvxfdAjigVFy1gfN
+nsmdyL0IeeZWi00qRQmTel0+mc9sqV4zczbxcHfDsSgrMvlDhiRxyusUyTcG1XHc
+20f/wqfTKDMkWYtWHB9Ty4BxmJUJZlixioim6eAk26L9STA1dB3092kYLk0/62Iv
+Fms1UjNn4DDERe/cUsaxAoIBAQDsqFbZ7R+SLBD/0HiQBWpwHbOjJ5AtTglNIrTZ
+XewLWwtYEvUT0bTfQDJJ+PBybR6RE4otFxKEfuR/WXQ9szQN6SUvIabaky/vRh0I
+q07NCBgpu3hFEw+qR4eXoXNFpYjSmAFyc1ViwwVfFvgqhywd45Rj46VkTv/x1UlS
+dOhuCUPTwuKVsDlRtPwBqi1dLCgdR0cEyjAlqe2DNrvtBmBlXqti4oFXDFPMzln8
+Xk2dyqDBeyKLnJU3W51x0J3hOl+axeMxPF1qWKgB5hvIADEKKywverMlyYqiWT+P
+QikUOocmiwna+sf2zEQrhGfa6Y8gWGMOcGy2NFRGAAyWkN/XAoIBAQDe/esmXmkq
+gKWSxqOzeJfUHVF4ygmvS7mKbDNMY3GCvZgfD+hpZeZARTNOu9JrDWiUcxT8Wj+j
+ZtYruQtCOxt1eiBoJOnYwaamm3d+HgvaE19+WxRGjcTGsZ6VreKzZlbgbAiCw/pd
+rDAqi+Bs7bNG4dOdSb9ctuj4YkBmpM6u5TQUCEKAIrr90gIJli/B9+x0PIgrovlm
+YCjlmrQlw+2DSFPrRxuqpXxNOyFyFsj992O/LFKAwYCyhtkqiUUqJbREIlyR1oL6
+X38Gbsp/jgrEUQ3N3773eV+bltz2nazEb67ps7nemjmZvjyluDjwfJdJJogWJsKK
+0jGC21EGqRGZAoIBAQCZoO5Aql5EVbbzWjHpzJo8Dgv/bj96KZ6AJHeiZAZHmOLU
+Wfoe05PHGbWLr77niU6+fyDEBKZQvM84nKmJJDw2i5NH9WCLo+EKQ0m1xv9wukB1
+Vu3MaYNR1v1+waBDJiKcE3FdCuHzKwbho9eWRAmvnX1HGxDS/TXJl9vxW1NHm1wc
+q/bLlYqgMA0oR4ELaw7fctX3lgmLabR16aI1TF5nb/1yQ/gSuj3sRkjEO7PHKzMQ
+Zw8V8qArP54FtJfJDkvh/XRvEfDSiJsIIIkIXJd5Mm2MpOqHLT6CBc3tAdYI+7Wg
+n8HWFdaZsCDQ3zNMOTJgnQAw72qjHXVXu9BwLbwhAoIBAE/PUXpKEBnGMXx22+BA
+KRch5yb0KMM0txNz5mhQry+769YyO1x9vAsEuXhUgNsP0X5QMhKfumchR0Ye1Ii/
+3vQM4cxkac3KgXrf6cSZvGQwytzOfFNEKklzCO9JbPoIhs+L2v/yZIliN1sC9TAH
+Y0LbUIHbA0KLtJYxlBsooVC3eAwzaJmz1HlD0LbdqfoiYd64S4RSsDCT+g8zb4aU
+uU1jdaWfradF01dQ8oeC4C0Ffg3OLzkmCInc+ZzfxIFxPTOlmLwZqocx5qTGwnMk
+w3XADNDCY/bu2ek19Z/Ojyc/UbsTOFMTn8oG7G3joX1xGjR0NgC3nqlQ0aekFzvr
+BwECggEBAKqlqm/yiPVViqHpQPrHbMWihaDEDF95L3jvvvByWS/96D7KUGGTHDJf
+Vszm8slfvdIrMtqdGH9DvhP2X1HDMLwl6zMFKgSwkf1bT0+A3a7TldJ4ezcoeAOd
+27YujmF5tk5yV2ufu9l0M+6bhXZuwdx1ktYo+GiR5G5lwllMT6ztQLcQA1BrSXgy
+1hr7OsKBBS6DGhjBxxULbp9DgADbemp8xXOXD5EhBIhu7TIioHoPDaXesjnsClda
+hRzyWycj5SDy1/kFAEYO94/wMFY5zYsmzUG3+AZ3eymdULP3mg6hkYJrht84R/hg
+YAxBUEWWyJC/Uhh1KnlWUPX3fcl0VyU=
+-----END PRIVATE KEY-----
diff --git a/tests/gold_tests/pluginTest/stek_share/stek_share.test.py b/tests/gold_tests/pluginTest/stek_share/stek_share.test.py
new file mode 100644
index 000000000..50f201002
--- /dev/null
+++ b/tests/gold_tests/pluginTest/stek_share/stek_share.test.py
@@ -0,0 +1,254 @@
+'''
+'''
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+
+Test.Summary = 'Test the STEK Share plugin'
+Test.testName = "stek_share"
+
+Test.SkipUnless(Condition.PluginExists('stek_share.so'))
+
+server = Test.MakeOriginServer('server')
+
+ts1 = Test.MakeATSProcess("ts1", select_ports=True, enable_tls=True)
+ts2 = Test.MakeATSProcess("ts2", select_ports=True, enable_tls=True)
+ts3 = Test.MakeATSProcess("ts3", select_ports=True, enable_tls=True)
+ts4 = Test.MakeATSProcess("ts4", select_ports=True, enable_tls=True)
+ts5 = Test.MakeATSProcess("ts5", select_ports=True, enable_tls=True)
+
+Test.Setup.Copy('ssl/self_signed.crt')
+Test.Setup.Copy('ssl/self_signed.key')
+Test.Setup.Copy('server_list.yaml')
+
+cert_path = os.path.join(Test.RunDirectory, 'self_signed.crt')
+key_path = os.path.join(Test.RunDirectory, 'self_signed.key')
+server_list_path = os.path.join(Test.RunDirectory, 'server_list.yaml')
+
+request_header1 = {
+ 'headers': 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n',
+ 'timestamp': '1469733493.993',
+ 'body': ''
+}
+response_header1 = {
+ 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n',
+ 'timestamp': '1469733493.993',
+ 'body': 'curl test'
+}
+server.addResponse('sessionlog.json', request_header1, response_header1)
+
+stek_share_conf_path_1 = os.path.join(ts1.Variables.CONFIGDIR, 'stek_share_conf.yaml')
+stek_share_conf_path_2 = os.path.join(ts2.Variables.CONFIGDIR, 'stek_share_conf.yaml')
+stek_share_conf_path_3 = os.path.join(ts3.Variables.CONFIGDIR, 'stek_share_conf.yaml')
+stek_share_conf_path_4 = os.path.join(ts4.Variables.CONFIGDIR, 'stek_share_conf.yaml')
+stek_share_conf_path_5 = os.path.join(ts5.Variables.CONFIGDIR, 'stek_share_conf.yaml')
+
+ts1.Disk.File(stek_share_conf_path_1, id="stek_share_conf_1", typename="ats:config")
+ts2.Disk.File(stek_share_conf_path_2, id="stek_share_conf_2", typename="ats:config")
+ts3.Disk.File(stek_share_conf_path_3, id="stek_share_conf_3", typename="ats:config")
+ts4.Disk.File(stek_share_conf_path_4, id="stek_share_conf_4", typename="ats:config")
+ts5.Disk.File(stek_share_conf_path_5, id="stek_share_conf_5", typename="ats:config")
+
+ts1.Disk.stek_share_conf_1.AddLines([
+ 'server_id: 1',
+ 'address: 127.0.0.1',
+ 'port: 10001',
+ 'asio_thread_pool_size: 4',
+ 'heart_beat_interval: 100',
+ 'election_timeout_lower_bound: 200',
+ 'election_timeout_upper_bound: 400',
+ 'reserved_log_items: 5',
+ 'snapshot_distance: 5',
+ 'client_req_timeout: 3000', # this is in milliseconds
+ 'key_update_interval: 3600', # this is in seconds
+ 'server_list_file: {0}'.format(server_list_path),
+ 'root_cert_file: {0}'.format(cert_path),
+ 'server_cert_file: {0}'.format(cert_path),
+ 'server_key_file: {0}'.format(key_path),
+ 'cert_verify_str: /C=US/ST=IL/O=Yahoo/OU=Edge/CN=stek-share'
+])
+
+ts2.Disk.stek_share_conf_2.AddLines([
+ 'server_id: 2',
+ 'address: 127.0.0.1',
+ 'port: 10002',
+ 'asio_thread_pool_size: 4',
+ 'heart_beat_interval: 100',
+ 'election_timeout_lower_bound: 200',
+ 'election_timeout_upper_bound: 400',
+ 'reserved_log_items: 5',
+ 'snapshot_distance: 5',
+ 'client_req_timeout: 3000', # this is in milliseconds
+ 'key_update_interval: 3600', # this is in seconds
+ 'server_list_file: {0}'.format(server_list_path),
+ 'root_cert_file: {0}'.format(cert_path),
+ 'server_cert_file: {0}'.format(cert_path),
+ 'server_key_file: {0}'.format(key_path),
+ 'cert_verify_str: /C=US/ST=IL/O=Yahoo/OU=Edge/CN=stek-share'
+])
+
+ts3.Disk.stek_share_conf_3.AddLines([
+ 'server_id: 3',
+ 'address: 127.0.0.1',
+ 'port: 10003',
+ 'asio_thread_pool_size: 4',
+ 'heart_beat_interval: 100',
+ 'election_timeout_lower_bound: 200',
+ 'election_timeout_upper_bound: 400',
+ 'reserved_log_items: 5',
+ 'snapshot_distance: 5',
+ 'client_req_timeout: 3000', # this is in milliseconds
+ 'key_update_interval: 3600', # this is in seconds
+ 'server_list_file: {0}'.format(server_list_path),
+ 'root_cert_file: {0}'.format(cert_path),
+ 'server_cert_file: {0}'.format(cert_path),
+ 'server_key_file: {0}'.format(key_path),
+ 'cert_verify_str: /C=US/ST=IL/O=Yahoo/OU=Edge/CN=stek-share'
+])
+
+ts4.Disk.stek_share_conf_4.AddLines([
+ 'server_id: 4',
+ 'address: 127.0.0.1',
+ 'port: 10004',
+ 'asio_thread_pool_size: 4',
+ 'heart_beat_interval: 100',
+ 'election_timeout_lower_bound: 200',
+ 'election_timeout_upper_bound: 400',
+ 'reserved_log_items: 5',
+ 'snapshot_distance: 5',
+ 'client_req_timeout: 3000', # this is in milliseconds
+ 'key_update_interval: 3600', # this is in seconds
+ 'server_list_file: {0}'.format(server_list_path),
+ 'root_cert_file: {0}'.format(cert_path),
+ 'server_cert_file: {0}'.format(cert_path),
+ 'server_key_file: {0}'.format(key_path),
+ 'cert_verify_str: /C=US/ST=IL/O=Yahoo/OU=Edge/CN=stek-share'
+])
+
+ts5.Disk.stek_share_conf_5.AddLines([
+ 'server_id: 5',
+ 'address: 127.0.0.1',
+ 'port: 10005',
+ 'asio_thread_pool_size: 4',
+ 'heart_beat_interval: 100',
+ 'election_timeout_lower_bound: 200',
+ 'election_timeout_upper_bound: 400',
+ 'reserved_log_items: 5',
+ 'snapshot_distance: 5',
+ 'client_req_timeout: 3000', # this is in milliseconds
+ 'key_update_interval: 3600', # this is in seconds
+ 'server_list_file: {0}'.format(server_list_path),
+ 'root_cert_file: {0}'.format(cert_path),
+ 'server_cert_file: {0}'.format(cert_path),
+ 'server_key_file: {0}'.format(key_path),
+ 'cert_verify_str: /C=US/ST=IL/O=Yahoo/OU=Edge/CN=stek-share'
+])
+
+ts1.Disk.records_config.update({'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'stek_share', 'proxy.config.exec_thread.autoconfig': 0, 'proxy.config.exec_thread.limit': 4, 'proxy.config.ssl.server.cert.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.server.private_key.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.session_cache': 2, 'proxy.config.ssl.session_cache.size': 1024, 'proxy.config.ssl.session_cache.timeout': 7200, 'proxy.config.ssl. [...]
+ 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA [...]
+ts1.Disk.plugin_config.AddLine('stek_share.so {0}'.format(stek_share_conf_path_1))
+ts1.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=self_signed.crt ssl_key_name=self_signed.key')
+ts1.Disk.remap_config.AddLine('map / http://127.0.0.1:{0}'.format(server.Variables.Port))
+
+ts2.Disk.records_config.update({'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'stek_share', 'proxy.config.exec_thread.autoconfig': 0, 'proxy.config.exec_thread.limit': 4, 'proxy.config.ssl.server.cert.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.server.private_key.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.session_cache': 2, 'proxy.config.ssl.session_cache.size': 1024, 'proxy.config.ssl.session_cache.timeout': 7200, 'proxy.config.ssl. [...]
+ 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA [...]
+ts2.Disk.plugin_config.AddLine('stek_share.so {0}'.format(stek_share_conf_path_2))
+ts2.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=self_signed.crt ssl_key_name=self_signed.key')
+ts2.Disk.remap_config.AddLine('map / http://127.0.0.1:{0}'.format(server.Variables.Port))
+
+ts3.Disk.records_config.update({'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'stek_share', 'proxy.config.exec_thread.autoconfig': 0, 'proxy.config.exec_thread.limit': 4, 'proxy.config.ssl.server.cert.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.server.private_key.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.session_cache': 2, 'proxy.config.ssl.session_cache.size': 1024, 'proxy.config.ssl.session_cache.timeout': 7200, 'proxy.config.ssl. [...]
+ 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA [...]
+ts3.Disk.plugin_config.AddLine('stek_share.so {0}'.format(stek_share_conf_path_3))
+ts3.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=self_signed.crt ssl_key_name=self_signed.key')
+ts3.Disk.remap_config.AddLine('map / http://127.0.0.1:{0}'.format(server.Variables.Port))
+
+ts4.Disk.records_config.update({'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'stek_share', 'proxy.config.exec_thread.autoconfig': 0, 'proxy.config.exec_thread.limit': 4, 'proxy.config.ssl.server.cert.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.server.private_key.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.session_cache': 2, 'proxy.config.ssl.session_cache.size': 1024, 'proxy.config.ssl.session_cache.timeout': 7200, 'proxy.config.ssl. [...]
+ 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA [...]
+ts4.Disk.plugin_config.AddLine('stek_share.so {0}'.format(stek_share_conf_path_4))
+ts4.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=self_signed.crt ssl_key_name=self_signed.key')
+ts4.Disk.remap_config.AddLine('map / http://127.0.0.1:{0}'.format(server.Variables.Port))
+
+ts5.Disk.records_config.update({'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'stek_share', 'proxy.config.exec_thread.autoconfig': 0, 'proxy.config.exec_thread.limit': 4, 'proxy.config.ssl.server.cert.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.server.private_key.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.session_cache': 2, 'proxy.config.ssl.session_cache.size': 1024, 'proxy.config.ssl.session_cache.timeout': 7200, 'proxy.config.ssl. [...]
+ 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA [...]
+ts5.Disk.plugin_config.AddLine('stek_share.so {0}'.format(stek_share_conf_path_5))
+ts5.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=self_signed.crt ssl_key_name=self_signed.key')
+ts5.Disk.remap_config.AddLine('map / http://127.0.0.1:{0}'.format(server.Variables.Port))
+
+
+def check_session(ev, test):
+ retval = False
+ f = open(test.GetContent(ev), 'r')
+ err = "Session ids match"
+ if not f:
+ err = "Failed to open {0}".format(openssl_output)
+ return (retval, "Check that session ids match", err)
+
+ content = f.read()
+ match = re.findall('Session-ID: ([0-9A-F]+)', content)
+
+ if match:
+ if all(i == j for i, j in zip(match, match[1:])):
+ err = "{0} reused successfully {1} times".format(match[0], len(match) - 1)
+ retval = True
+ else:
+ err = "Session is not being reused as expected"
+ else:
+ err = "Didn't find session id"
+ return (retval, "Check that session ids match", err)
+
+
+tr1 = Test.AddTestRun('Basic Curl test, and give it enough time for all ATS to start up and sync STEK')
+tr1.Processes.Default.Command = 'sleep 10 && curl https://127.0.0.1:{0} -k'.format(ts1.Variables.ssl_port)
+tr1.Processes.Default.ReturnCode = 0
+tr1.Processes.Default.StartBefore(server)
+tr1.Processes.Default.StartBefore(ts1)
+tr1.Processes.Default.StartBefore(ts2)
+tr1.Processes.Default.StartBefore(ts3)
+tr1.Processes.Default.StartBefore(ts4)
+tr1.Processes.Default.StartBefore(ts5)
+tr1.Processes.Default.Streams.All = Testers.ContainsExpression('curl test', 'Making sure the basics still work')
+ts1.Streams.All = Testers.ContainsExpression('Generate initial STEK succeeded', 'should succeed')
+ts2.Streams.All = Testers.ContainsExpression('Generate initial STEK succeeded', 'should succeed')
+ts3.Streams.All = Testers.ContainsExpression('Generate initial STEK succeeded', 'should succeed')
+ts4.Streams.All = Testers.ContainsExpression('Generate initial STEK succeeded', 'should succeed')
+ts5.Streams.All = Testers.ContainsExpression('Generate initial STEK succeeded', 'should succeed')
+tr1.StillRunningAfter = server
+tr1.StillRunningAfter += ts1
+tr1.StillRunningAfter += ts2
+tr1.StillRunningAfter += ts3
+tr1.StillRunningAfter += ts4
+tr1.StillRunningAfter += ts5
+
+tr2 = Test.AddTestRun("TLSv1.2 Session Ticket")
+tr2.Command = \
+ 'echo -e "GET / HTTP/1.1\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{0} -sess_out {5} && ' \
+ 'echo -e "GET / HTTP/1.1\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{0} -sess_in {5} && ' \
+ 'echo -e "GET / HTTP/1.1\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{1} -sess_in {5} && ' \
+ 'echo -e "GET / HTTP/1.1\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{2} -sess_in {5} && ' \
+ 'echo -e "GET / HTTP/1.1\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{3} -sess_in {5} && ' \
+ 'echo -e "GET / HTTP/1.1\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{4} -sess_in {5}' \
+ .format(
+ ts1.Variables.ssl_port,
+ ts2.Variables.ssl_port,
+ ts3.Variables.ssl_port,
+ ts4.Variables.ssl_port,
+ ts5.Variables.ssl_port,
+ os.path.join(Test.RunDirectory, 'sess.dat')
+ )
+tr2.ReturnCode = 0
+tr2.Processes.Default.Streams.All.Content = Testers.Lambda(check_session)