You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by zw...@apache.org on 2020/07/08 15:25:08 UTC

[trafficserver] branch 9.0.x updated (513a80e -> 3385e27)

This is an automated email from the ASF dual-hosted git repository.

zwoop pushed a change to branch 9.0.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git.


    from 513a80e  Log whether client certs were exchanged in TLS handshake (#6699)
     new 137ea61  Commenting EventIO methods (#6712)
     new 368c88f  Fix support for openssl async engine (#6910)
     new 3385e27  Assert on valid boundaries for UserArgTable access (#6953)

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 contrib/openssl/README.md                          |  2 +-
 contrib/openssl/async_engine.c                     | 45 +++++++---
 include/tscore/PluginUserArgs.h                    |  4 +-
 iocore/net/P_UnixNet.h                             | 98 +++++++++++++++++-----
 iocore/net/SSLNetVConnection.cc                    | 26 +++---
 iocore/net/SSLUtils.cc                             |  7 +-
 .../tls/{tls.test.py => tls_engine.test.py}        | 70 ++++++++++------
 7 files changed, 175 insertions(+), 77 deletions(-)
 copy tests/gold_tests/tls/{tls.test.py => tls_engine.test.py} (59%)


[trafficserver] 03/03: Assert on valid boundaries for UserArgTable access (#6953)

Posted by zw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zwoop pushed a commit to branch 9.0.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git

commit 3385e2787d8e27d893a8255c3003df50bad78356
Author: Sudheer Vinukonda <su...@apache.org>
AuthorDate: Mon Jun 29 16:36:39 2020 -0700

    Assert on valid boundaries for UserArgTable access (#6953)
    
    Note the index is size_t (unsigned) so it is always
    > 0
    
    (cherry picked from commit 524d2e4c2434ff6a22a6b5d3b100367b662d175a)
---
 include/tscore/PluginUserArgs.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/tscore/PluginUserArgs.h b/include/tscore/PluginUserArgs.h
index 8a6a314..057e37c 100644
--- a/include/tscore/PluginUserArgs.h
+++ b/include/tscore/PluginUserArgs.h
@@ -53,14 +53,14 @@ public:
   void *
   get_user_arg(size_t ix) const
   {
-    ink_assert(ix < user_args.size());
+    ink_release_assert(ix < user_args.size());
     return this->user_args[ix];
   };
 
   void
   set_user_arg(size_t ix, void *arg)
   {
-    ink_assert(ix < user_args.size());
+    ink_release_assert(ix < user_args.size());
     user_args[ix] = arg;
   };
 


[trafficserver] 02/03: Fix support for openssl async engine (#6910)

Posted by zw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zwoop pushed a commit to branch 9.0.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git

commit 368c88f77924faae32385df510a5bcf8a602de33
Author: Susan Hinrichs <sh...@yahoo-inc.com>
AuthorDate: Wed Jul 1 10:54:09 2020 -0500

    Fix support for openssl async engine (#6910)
    
    (cherry picked from commit c32fc645ea44ba0615461d10057ef5a8e9516b29)
---
 contrib/openssl/README.md               |  2 +-
 contrib/openssl/async_engine.c          | 45 ++++++++++-----
 iocore/net/P_UnixNet.h                  | 51 +++++++++++------
 iocore/net/SSLNetVConnection.cc         | 26 +++++----
 iocore/net/SSLUtils.cc                  |  7 ++-
 tests/gold_tests/tls/tls_engine.test.py | 98 +++++++++++++++++++++++++++++++++
 6 files changed, 185 insertions(+), 44 deletions(-)

diff --git a/contrib/openssl/README.md b/contrib/openssl/README.md
index 428f9a6..faa89c7 100644
--- a/contrib/openssl/README.md
+++ b/contrib/openssl/README.md
@@ -5,6 +5,6 @@ It should be built as follows.  It must be build against openssl 1.1 or better f
 
 gcc -fPIC -shared -g -o async-test.so -I<path to openssl headers> -L<path to openssl library> -lssl -lcrypto -lpthread async_engine.c
 
-load_engine.cnf is an example openssl config file that can be passed to Traffic Server via the proxy.config.ssl.engine_cnf_file setting.
+load_engine.cnf is an example openssl config file that can be passed to Traffic Server via the proxy.config.ssl.engine.conf_file setting.
 It describes which crypto engines should be loaded and how they should be used.  In the case of our async-test crypto engine it will be used for
 RSA operations
diff --git a/contrib/openssl/async_engine.c b/contrib/openssl/async_engine.c
index facffe0..7c3604d 100644
--- a/contrib/openssl/async_engine.c
+++ b/contrib/openssl/async_engine.c
@@ -72,6 +72,15 @@ static int async_rsa_finish(RSA *rsa);
 
 static RSA_METHOD *async_rsa_method = NULL;
 
+EVP_PKEY *async_load_privkey(ENGINE *e, const char *s_key_id, UI_METHOD *ui_method, void *callback_data)
+{
+  printf("Loading key %s\n", s_key_id);
+  FILE *f = fopen(s_key_id, "r");
+  EVP_PKEY *key = PEM_read_PrivateKey(f, NULL, NULL, NULL);
+  fclose(f);
+  return key;
+}
+
 static int bind_async(ENGINE *e)
 {
     /* Setup RSA_METHOD */
@@ -96,7 +105,8 @@ static int bind_async(ENGINE *e)
         || !ENGINE_set_RSA(e, async_rsa_method)
         || !ENGINE_set_destroy_function(e, async_destroy)
         || !ENGINE_set_init_function(e, engine_async_init)
-        || !ENGINE_set_finish_function(e, async_finish)) {
+        || !ENGINE_set_finish_function(e, async_finish)
+        || !ENGINE_set_load_privkey_function(e, async_load_privkey)) {
         fprintf(stderr, "Failed to initialize\n");
         return 0;
     }
@@ -176,14 +186,17 @@ static void async_pause_job(void) {
     OSSL_ASYNC_FD *writefd;
     char buf = DUMMY_CHAR;
 
-    if ((job = ASYNC_get_current_job()) == NULL)
+    if ((job = ASYNC_get_current_job()) == NULL) {
+        printf("No job\n");
         return;
+    }
 
     waitctx = ASYNC_get_wait_ctx(job);
 
     if (ASYNC_WAIT_CTX_get_fd(waitctx, engine_id, &pipefds[0],
                               (void **)&writefd)) {
-        pipefds[1] = *writefd;
+        printf("Existing wait ctx\n");
+        return;
     } else {
         writefd = (OSSL_ASYNC_FD *)OPENSSL_malloc(sizeof(*writefd));
         if (writefd == NULL)
@@ -194,6 +207,8 @@ static void async_pause_job(void) {
         }
         *writefd = pipefds[1];
 
+        printf("New wait ctx %d %d\n", pipefds[0], pipefds[1]);
+
         if(!ASYNC_WAIT_CTX_set_wait_fd(waitctx, engine_id, pipefds[0],
                                        writefd, wait_cleanup)) {
             wait_cleanup(waitctx, engine_id, pipefds[0], writefd);
@@ -213,9 +228,11 @@ void *
 delay_method(void *arg)
 {
   int signal_fd = (intptr_t)arg;
-  sleep(5);
-  uint64_t buf = 1;
+  sleep(2);
+  char buf = DUMMY_CHAR;
   write(signal_fd, &buf, sizeof(buf));
+  printf("Send signal to %d\n", signal_fd);
+  return NULL;
 }
 
 
@@ -223,25 +240,27 @@ void
 spawn_delay_thread()
 {
   pthread_t thread_id;
-  OSSL_ASYNC_FD signal_fd;
-  OSSL_ASYNC_FD pipefds[2] = {0, 0};
   ASYNC_JOB *job;
-  if ((job = ASYNC_get_current_job()) == NULL)
+  if ((job = ASYNC_get_current_job()) == NULL) {
+      printf("Spawn no job\n");
       return;
+  }
 
   ASYNC_WAIT_CTX *waitctx = ASYNC_get_wait_ctx(job);
 
   size_t numfds;
-  if (ASYNC_WAIT_CTX_get_all_fds(waitctx, &signal_fd, &numfds) && numfds > 0) {
+  if (ASYNC_WAIT_CTX_get_all_fds(waitctx, NULL, &numfds) && numfds > 0) {
+    printf("Spawn, wait_ctx exists.  Go away, something else is using this job\n");
   } else {
+    OSSL_ASYNC_FD signal_fd;
     OSSL_ASYNC_FD pipefds[2] = {0,0};
     OSSL_ASYNC_FD *writefd = OPENSSL_malloc(sizeof(*writefd));
     pipe(pipefds);
     signal_fd = *writefd = pipefds[1];
     ASYNC_WAIT_CTX_set_wait_fd(waitctx, engine_id, pipefds[0], writefd, wait_cleanup);
+    printf("Spawn, create wait_ctx %d %d\n", pipefds[0], pipefds[1]);
+    pthread_create(&thread_id, NULL, delay_method, (void *)((intptr_t)signal_fd));
   }
-
-  pthread_create(&thread_id, NULL, delay_method, (void *)((intptr_t)signal_fd));
 }
 
 
@@ -264,7 +283,7 @@ static int async_pub_dec(int flen, const unsigned char *from,
 static int async_rsa_priv_enc(int flen, const unsigned char *from,
                       unsigned char *to, RSA *rsa, int padding)
 {
-    //printf("async_priv_enc\n");
+    printf("async_priv_enc\n");
     spawn_delay_thread();
     async_pause_job();
     return RSA_meth_get_priv_enc(RSA_PKCS1_OpenSSL())
@@ -274,7 +293,7 @@ static int async_rsa_priv_enc(int flen, const unsigned char *from,
 static int async_rsa_priv_dec(int flen, const unsigned char *from,
                       unsigned char *to, RSA *rsa, int padding)
 {
-    //printf("async_priv_dec\n");
+    printf("async_priv_dec\n");
     spawn_delay_thread();
     async_pause_job();
     return RSA_meth_get_priv_dec(RSA_PKCS1_OpenSSL())
diff --git a/iocore/net/P_UnixNet.h b/iocore/net/P_UnixNet.h
index be0ae32..2c74a3e 100644
--- a/iocore/net/P_UnixNet.h
+++ b/iocore/net/P_UnixNet.h
@@ -97,18 +97,21 @@ struct EventIO {
     UnixUDPConnection *uc;
   } data; ///< a kind of continuation
 
-  int start(EventLoop l, DNSConnection *vc, int events);
-  int start(EventLoop l, NetAccept *vc, int events);
-  int start(EventLoop l, NetEvent *ne, int events);
-  int start(EventLoop l, UnixUDPConnection *vc, int events);
-  /** Setup a continuation to be called when a file descriptor is available for read or write.
+  /** The start methods all logically Setup a class to be called
+     when a file descriptor is available for read or write.
+     The type of the classes vary.  Generally the file descriptor
+     is pulled from the class, but there is one option that lets
+     the file descriptor be expressed directly.
      @param l the event loop
-     @param fd file descriptor (or port)
-     @param c the continuation to call
      @param events a mask of flags (for details `man epoll_ctl`)
      @return int the number of events created, -1 is error
    */
-  int start(EventLoop l, int fd, Continuation *c, int events);
+  int start(EventLoop l, DNSConnection *vc, int events);
+  int start(EventLoop l, NetAccept *vc, int events);
+  int start(EventLoop l, NetEvent *ne, int events);
+  int start(EventLoop l, UnixUDPConnection *vc, int events);
+  int start(EventLoop l, int fd, NetEvent *ne, int events);
+  int start_common(EventLoop l, int fd, int events);
 
   /** Alter the events that will trigger the continuation, for level triggered I/O.
      @param events add with positive mask(+EVENTIO_READ), or remove with negative mask (-EVENTIO_READ)
@@ -577,27 +580,33 @@ write_disable(NetHandler *nh, NetEvent *ne)
 TS_INLINE int
 EventIO::start(EventLoop l, DNSConnection *vc, int events)
 {
-  type = EVENTIO_DNS_CONNECTION;
-  return start(l, vc->fd, (Continuation *)vc, events);
+  type        = EVENTIO_DNS_CONNECTION;
+  data.dnscon = vc;
+  return start_common(l, vc->fd, events);
 }
 TS_INLINE int
 EventIO::start(EventLoop l, NetAccept *vc, int events)
 {
-  type = EVENTIO_NETACCEPT;
-  return start(l, vc->server.fd, (Continuation *)vc, events);
+  type    = EVENTIO_NETACCEPT;
+  data.na = vc;
+  return start_common(l, vc->server.fd, events);
 }
 TS_INLINE int
 EventIO::start(EventLoop l, NetEvent *ne, int events)
 {
-  type = EVENTIO_READWRITE_VC;
-  return start(l, ne->get_fd(), (Continuation *)ne, events);
+  type    = EVENTIO_READWRITE_VC;
+  data.ne = ne;
+  return start_common(l, ne->get_fd(), events);
 }
+
 TS_INLINE int
 EventIO::start(EventLoop l, UnixUDPConnection *vc, int events)
 {
-  type = EVENTIO_UDP_CONNECTION;
-  return start(l, vc->fd, (Continuation *)vc, events);
+  type    = EVENTIO_UDP_CONNECTION;
+  data.uc = vc;
+  return start_common(l, vc->fd, events);
 }
+
 TS_INLINE int
 EventIO::close()
 {
@@ -624,13 +633,19 @@ EventIO::close()
 }
 
 TS_INLINE int
-EventIO::start(EventLoop l, int afd, Continuation *c, int e)
+EventIO::start(EventLoop l, int afd, NetEvent *ne, int e)
+{
+  data.ne = ne;
+  return start_common(l, afd, e);
+}
+
+TS_INLINE int
+EventIO::start_common(EventLoop l, int afd, int e)
 {
   if (!this->syscall) {
     return 0;
   }
 
-  data.c     = c;
   fd         = afd;
   event_loop = l;
 #if TS_USE_EPOLL
diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc
index 8e8976c..2f641e9 100644
--- a/iocore/net/SSLNetVConnection.cc
+++ b/iocore/net/SSLNetVConnection.cc
@@ -1233,18 +1233,22 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err)
 #if TS_USE_TLS_ASYNC
   if (ssl_error == SSL_ERROR_WANT_ASYNC) {
     size_t numfds;
-    OSSL_ASYNC_FD waitfd;
+    OSSL_ASYNC_FD *waitfds;
     // Set up the epoll entry for the signalling
-    if (SSL_get_all_async_fds(ssl, &waitfd, &numfds) && numfds > 0) {
-      // Temporarily disable regular net
-      read_disable(nh, this);
-      this->ep.stop(); // Modify used in read_disable doesn't work for edge triggered epol
-      // Have to have the read NetState enabled because we are using it for the signal vc
-      read.enabled = true;
-      write_disable(nh, this);
-      PollDescriptor *pd = get_PollDescriptor(this_ethread());
-      this->ep.start(pd, waitfd, this, EVENTIO_READ);
-      this->ep.type = EVENTIO_READWRITE_VC;
+    if (SSL_get_all_async_fds(ssl, nullptr, &numfds) && numfds > 0) {
+      // Allocate space for the waitfd on the stack, should only be one most all of the time
+      waitfds = reinterpret_cast<OSSL_ASYNC_FD *>(alloca(sizeof(OSSL_ASYNC_FD) * numfds));
+      if (SSL_get_all_async_fds(ssl, waitfds, &numfds) && numfds > 0) {
+        // Temporarily disable regular net
+        this->read.triggered  = false;
+        this->write.triggered = false;
+        this->ep.stop(); // Modify used in read_disable doesn't work for edge triggered epol
+        // Have to have the read NetState enabled because we are using it for the signal vc
+        read.enabled       = true;
+        PollDescriptor *pd = get_PollDescriptor(this_ethread());
+        this->ep.start(pd, waitfds[0], static_cast<NetEvent *>(this), EVENTIO_READ);
+        this->ep.type = EVENTIO_READWRITE_VC;
+      }
     }
   } else if (SSLConfigParams::async_handshake_enabled) {
     // Clean up the epoll entry for signalling
diff --git a/iocore/net/SSLUtils.cc b/iocore/net/SSLUtils.cc
index db0fea4..9965120 100644
--- a/iocore/net/SSLUtils.cc
+++ b/iocore/net/SSLUtils.cc
@@ -921,7 +921,12 @@ SSLPrivateKeyHandler(SSL_CTX *ctx, const SSLConfigParams *params, const std::str
 #ifndef OPENSSL_IS_BORINGSSL
   ENGINE *e = ENGINE_get_default_RSA();
   if (e != nullptr) {
-    const char *argkey = (keyPath == nullptr || keyPath[0] == '\0') ? completeServerCertPath.c_str() : keyPath;
+    ats_scoped_str argkey;
+    if (keyPath == nullptr || keyPath[0] == '\0') {
+      argkey = completeServerCertPath.c_str();
+    } else {
+      argkey = Layout::get()->relative_to(params->serverKeyPathOnly, keyPath);
+    }
     if (!SSL_CTX_use_PrivateKey(ctx, ENGINE_load_private_key(e, argkey, nullptr, nullptr))) {
       SSLError("failed to load server private key from engine");
     }
diff --git a/tests/gold_tests/tls/tls_engine.test.py b/tests/gold_tests/tls/tls_engine.test.py
new file mode 100644
index 0000000..bdfe946
--- /dev/null
+++ b/tests/gold_tests/tls/tls_engine.test.py
@@ -0,0 +1,98 @@
+'''
+'''
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+import os
+
+
+# Someday should add client cert to origin to exercise the
+# engine interface on the other side
+
+Test.Summary = '''
+Test tls via the async interface with the sample async_engine
+'''
+
+Test.SkipUnless(Condition.HasOpenSSLVersion('1.1.1'))
+
+# Define default ATS
+ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True)
+server = Test.MakeOriginServer("server")
+
+# Compile with tsxs.  That should bring in the consisten versions of openssl
+ts.Setup.Copy(os.path.join(Test.Variables.AtsTestToolsDir, '../../contrib/openssl', 'async_engine.c'), Test.RunDirectory)
+ts.Setup.RunCommand("tsxs -o async_engine.so async_engine.c")
+
+# Add info the origin server responses
+server.addResponse("sessionlog.json",
+                   {"headers": "GET / HTTP/1.1\r\nuuid: basic\r\n\r\n", "timestamp": "1469733493.993", "body": ""},
+                   {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\nCache-Control: max-age=3600\r\nContent-Length: 2\r\n\r\n", "timestamp": "1469733493.993", "body": "ok"})
+
+# add ssl materials like key, certificates for the server
+ts.addSSLfile("ssl/server.pem")
+ts.addSSLfile("ssl/server.key")
+
+ts.Disk.remap_config.AddLine(
+    'map / http://127.0.0.1:{0}'.format(server.Variables.Port)
+)
+
+ts.Disk.ssl_multicert_config.AddLine(
+    'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key'
+)
+ts.Disk.records_config.update({
+    'proxy.config.ssl.server.cert.path': '{0}'.format(ts.Variables.SSLDir),
+    'proxy.config.ssl.server.private_key.path': '{0}'.format(ts.Variables.SSLDir),
+    'proxy.config.ssl.client.verify.server':  0,
+    'proxy.config.exec_thread.autoconfig.scale': 1.0,
+    'proxy.config.ssl.server.cipher_suite': 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES256-GCM-SHA384:ECDHE-RSA-RC4-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:RC4-SHA:RC4-MD5:AES128-SHA:AES256-SHA:DES-CBC3-SHA!SRP:!DSS:!PSK:!aNULL:!eNULL:!SSLv2',
+    'proxy.config.ssl.engine.conf_file': '{0}/ts/config/load_engine.cnf'.format(Test.RunDirectory),
+    'proxy.config.ssl.async.handshake.enabled': 1,
+    'proxy.config.diags.debug.enabled': 0,
+    'proxy.config.diags.debug.tags': 'ssl'
+})
+
+ts.Disk.MakeConfigFile('load_engine.cnf').AddLines([
+    'openssl_conf = openssl_init',
+    '',
+    '[openssl_init]',
+    '',
+    'engines = engine_section',
+    '',
+    '[engine_section]',
+    '',
+    'async = async_section',
+    '',
+    '[async_section]',
+    '',
+    'dynamic_path = {0}/async_engine.so'.format(Test.RunDirectory),
+    '',
+    'engine_id = async-test',
+    '',
+    'default_algorithms = RSA',
+    '',
+    'init = 1'])
+
+# Make a basic request.  Hopefully it goes through
+tr = Test.AddTestRun("Run-Test")
+tr.Processes.Default.Command = "curl -k -v -H uuid:basic -H host:example.com  https://127.0.0.1:{0}/".format(ts.Variables.ssl_port)
+tr.ReturnCode = 0
+tr.Processes.Default.StartBefore(server)
+tr.Processes.Default.StartBefore(Test.Processes.ts, ready=When.PortOpen(ts.Variables.ssl_port))
+tr.Processes.Default.Streams.All = Testers.ContainsExpression("HTTP/(2|1\.1) 200", "Request succeeds")
+tr.StillRunningAfter = server
+
+ts.Streams.All += Testers.ContainsExpression("Send signal to ", "The Async engine triggers")


[trafficserver] 01/03: Commenting EventIO methods (#6712)

Posted by zw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

zwoop pushed a commit to branch 9.0.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git

commit 137ea6192b48a21a9bb1f462146d9b72d8f36f65
Author: a-a-ron <ac...@verizonmedia.com>
AuthorDate: Fri May 1 11:12:06 2020 -0500

    Commenting EventIO methods (#6712)
    
    * Commenting EventIO methods
    
    (cherry picked from commit 654fbe6a6794be4ab060666ed909dcbebeee82ec)
---
 iocore/net/P_UnixNet.h | 61 ++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 49 insertions(+), 12 deletions(-)

diff --git a/iocore/net/P_UnixNet.h b/iocore/net/P_UnixNet.h
index fbc10ec..be0ae32 100644
--- a/iocore/net/P_UnixNet.h
+++ b/iocore/net/P_UnixNet.h
@@ -79,33 +79,55 @@ class NetEvent;
 class UnixUDPConnection;
 struct DNSConnection;
 struct NetAccept;
+
+/// Unified API for setting and clearing kernel and epoll events.
 struct EventIO {
-  int fd = -1;
+  int fd = -1; ///< file descriptor, often a system port
 #if TS_USE_KQUEUE || TS_USE_EPOLL && !defined(USE_EDGE_TRIGGER) || TS_USE_PORT
-  int events = 0;
+  int events = 0; ///< a bit mask of enabled events
 #endif
-  EventLoop event_loop = nullptr;
-  bool syscall         = true;
-  int type             = 0;
+  EventLoop event_loop = nullptr; ///< the assigned event loop
+  bool syscall         = true;    ///< if false, disable all functionality (for QUIC)
+  int type             = 0;       ///< class identifier of union data.
   union {
     Continuation *c;
     NetEvent *ne;
     DNSConnection *dnscon;
     NetAccept *na;
     UnixUDPConnection *uc;
-  } data;
+  } data; ///< a kind of continuation
+
   int start(EventLoop l, DNSConnection *vc, int events);
   int start(EventLoop l, NetAccept *vc, int events);
   int start(EventLoop l, NetEvent *ne, int events);
   int start(EventLoop l, UnixUDPConnection *vc, int events);
+  /** Setup a continuation to be called when a file descriptor is available for read or write.
+     @param l the event loop
+     @param fd file descriptor (or port)
+     @param c the continuation to call
+     @param events a mask of flags (for details `man epoll_ctl`)
+     @return int the number of events created, -1 is error
+   */
   int start(EventLoop l, int fd, Continuation *c, int events);
-  // Change the existing events by adding modify(EVENTIO_READ)
-  // or removing modify(-EVENTIO_READ), for level triggered I/O
+
+  /** Alter the events that will trigger the continuation, for level triggered I/O.
+     @param events add with positive mask(+EVENTIO_READ), or remove with negative mask (-EVENTIO_READ)
+     @return int the number of events created, -1 is error
+   */
   int modify(int events);
-  // Refresh the existing events (i.e. KQUEUE EV_CLEAR), for edge triggered I/O
+
+  /** Refresh the existing events (i.e. KQUEUE EV_CLEAR), for edge triggered I/O
+     @param events mask of events
+     @return int the number of events created, -1 is error
+   */
   int refresh(int events);
+
+  /// Remove the kernal or epoll event. Returns 0 on success.
   int stop();
+
+  /// Remove the epoll event and close the connection. Returns 0 on success.
   int close();
+
   EventIO() { data.c = nullptr; }
 };
 
@@ -510,9 +532,15 @@ check_transient_accept_error(int res)
   }
 }
 
-//
-// Disable a NetEvent
-//
+/** Disable reading on the NetEvent @a ne.
+     @param nh Nethandler that owns @a ne.
+     @param ne The @c NetEvent to modify.
+
+     - If write is already disable, also disable the inactivity timeout.
+     - clear read enabled flag.
+     - Remove the @c epoll READ flag.
+     - Take @a ne out of the read ready list.
+*/
 static inline void
 read_disable(NetHandler *nh, NetEvent *ne)
 {
@@ -525,6 +553,15 @@ read_disable(NetHandler *nh, NetEvent *ne)
   ne->ep.modify(-EVENTIO_READ);
 }
 
+/** Disable writing on the NetEvent @a ne.
+     @param nh Nethandler that owns @a ne.
+     @param ne The @c NetEvent to modify.
+
+     - If read is already disable, also disable the inactivity timeout.
+     - clear write enabled flag.
+     - Remove the @c epoll WRITE flag.
+     - Take @a ne out of the write ready list.
+*/
 static inline void
 write_disable(NetHandler *nh, NetEvent *ne)
 {