You are viewing a plain text version of this content. The canonical link for it is here.
Posted to cvs@httpd.apache.org by ic...@apache.org on 2015/11/20 16:13:11 UTC

svn commit: r1715371 [4/6] - in /httpd/httpd/branches/2.4.x: ./ docs/manual/mod/ modules/http2/

Modified: httpd/httpd/branches/2.4.x/modules/http2/h2_session.c
URL: http://svn.apache.org/viewvc/httpd/httpd/branches/2.4.x/modules/http2/h2_session.c?rev=1715371&r1=1715370&r2=1715371&view=diff
==============================================================================
--- httpd/httpd/branches/2.4.x/modules/http2/h2_session.c (original)
+++ httpd/httpd/branches/2.4.x/modules/http2/h2_session.c Fri Nov 20 15:13:11 2015
@@ -24,9 +24,13 @@
 #include <http_log.h>
 
 #include "h2_private.h"
+#include "h2_bucket_eoc.h"
+#include "h2_bucket_eos.h"
 #include "h2_config.h"
 #include "h2_h2.h"
 #include "h2_mplx.h"
+#include "h2_push.h"
+#include "h2_request.h"
 #include "h2_response.h"
 #include "h2_stream.h"
 #include "h2_stream_set.h"
@@ -41,57 +45,110 @@ static int frame_print(const nghttp2_fra
 
 static int h2_session_status_from_apr_status(apr_status_t rv)
 {
-    switch (rv) {
-        case APR_SUCCESS:
-            return NGHTTP2_NO_ERROR;
-        case APR_EAGAIN:
-        case APR_TIMEUP:
-            return NGHTTP2_ERR_WOULDBLOCK;
-        case APR_EOF:
+    if (rv == APR_SUCCESS) {
+        return NGHTTP2_NO_ERROR;
+    }
+    else if (APR_STATUS_IS_EAGAIN(rv)) {
+        return NGHTTP2_ERR_WOULDBLOCK;
+    }
+    else if (APR_STATUS_IS_EOF(rv)) {
             return NGHTTP2_ERR_EOF;
-        default:
-            return NGHTTP2_ERR_PROTO;
     }
+    return NGHTTP2_ERR_PROTO;
 }
 
-static int stream_open(h2_session *session, int stream_id)
+h2_stream *h2_session_open_stream(h2_session *session, int stream_id)
 {
     h2_stream * stream;
+    apr_pool_t *stream_pool;
     if (session->aborted) {
-        return NGHTTP2_ERR_CALLBACK_FAILURE;
+        return NULL;
     }
     
-    stream = h2_mplx_open_io(session->mplx, stream_id);
-    if (stream) {
-        h2_stream_set_add(session->streams, stream);
-        if (stream->id > session->max_stream_received) {
-            session->max_stream_received = stream->id;
-        }
-        
-        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
-                      "h2_session: stream(%ld-%d): opened",
-                      session->id, stream_id);
+    if (session->spare) {
+        stream_pool = session->spare;
+        session->spare = NULL;
+    }
+    else {
+        apr_pool_create(&stream_pool, session->pool);
+    }
+    
+    stream = h2_stream_open(stream_id, stream_pool, session);
+    
+    h2_stream_set_add(session->streams, stream);
+    if (H2_STREAM_CLIENT_INITIATED(stream_id)
+        && stream_id > session->max_stream_received) {
+        session->max_stream_received = stream->id;
+    }
+    
+    return stream;
+}
+
+apr_status_t h2_session_flush(h2_session *session) 
+{
+    return h2_conn_io_flush(&session->io);
+}
+
+/**
+ * Determine the importance of streams when scheduling tasks.
+ * - if both stream depend on the same one, compare weights
+ * - if one stream is closer to the root, prioritize that one
+ * - if both are on the same level, use the weight of their root
+ *   level ancestors
+ */
+static int spri_cmp(int sid1, nghttp2_stream *s1, 
+                    int sid2, nghttp2_stream *s2, h2_session *session)
+{
+    nghttp2_stream *p1, *p2;
+    
+    p1 = nghttp2_stream_get_parent(s1);
+    p2 = nghttp2_stream_get_parent(s2);
+    
+    if (p1 == p2) {
+        int32_t w1, w2;
         
-        return 0;
+        w1 = nghttp2_stream_get_weight(s1);
+        w2 = nghttp2_stream_get_weight(s2);
+        return w2 - w1;
+    }
+    else if (!p1) {
+        /* stream 1 closer to root */
+        return -1;
     }
+    else if (!p2) {
+        /* stream 2 closer to root */
+        return 1;
+    }
+    return spri_cmp(sid1, p1, sid2, p2, session);
+}
+
+static int stream_pri_cmp(int sid1, int sid2, void *ctx)
+{
+    h2_session *session = ctx;
+    nghttp2_stream *s1, *s2;
     
-    ap_log_cerror(APLOG_MARK, APLOG_ERR, APR_ENOMEM, session->c,
-                  APLOGNO(02918) 
-                  "h2_session: stream(%ld-%d): unable to create",
-                  session->id, stream_id);
-    return NGHTTP2_ERR_INVALID_STREAM_ID;
+    s1 = nghttp2_session_find_stream(session->ngh2, sid1);
+    s2 = nghttp2_session_find_stream(session->ngh2, sid2);
+
+    if (s1 == s2) {
+        return 0;
+    }
+    else if (!s1) {
+        return 1;
+    }
+    else if (!s2) {
+        return -1;
+    }
+    return spri_cmp(sid1, s1, sid2, s2, session);
 }
 
-static apr_status_t stream_end_headers(h2_session *session,
-                                       h2_stream *stream, int eos)
+static apr_status_t stream_schedule(h2_session *session,
+                                    h2_stream *stream, int eos)
 {
     (void)session;
-    return h2_stream_write_eoh(stream, eos);
+    return h2_stream_schedule(stream, eos, stream_pri_cmp, session);
 }
 
-static apr_status_t send_data(h2_session *session, const char *data, 
-                              apr_size_t length);
-
 /*
  * Callback when nghttp2 wants to send bytes back to the client.
  */
@@ -100,14 +157,15 @@ static ssize_t send_cb(nghttp2_session *
                        int flags, void *userp)
 {
     h2_session *session = (h2_session *)userp;
-    apr_status_t status = send_data(session, (const char *)data, length);
+    apr_status_t status;
     
     (void)ngh2;
     (void)flags;
+    status = h2_conn_io_write(&session->io, (const char *)data, length);
     if (status == APR_SUCCESS) {
         return length;
     }
-    if (status == APR_EAGAIN || status == APR_TIMEUP) {
+    if (APR_STATUS_IS_EAGAIN(status)) {
         return NGHTTP2_ERR_WOULDBLOCK;
     }
     ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, session->c,
@@ -140,19 +198,19 @@ static int on_data_chunk_recv_cb(nghttp2
                                  int32_t stream_id,
                                  const uint8_t *data, size_t len, void *userp)
 {
-    int rv;
     h2_session *session = (h2_session *)userp;
+    apr_status_t status = APR_SUCCESS;
     h2_stream * stream;
-    apr_status_t status;
+    int rv;
     
     (void)flags;
     if (session->aborted) {
         return NGHTTP2_ERR_CALLBACK_FAILURE;
     }
-    stream = h2_stream_set_get(session->streams, stream_id);
+    
+    stream = h2_session_get_stream(session, stream_id);
     if (!stream) {
-        ap_log_cerror(APLOG_MARK, APLOG_ERR, 0, session->c,
-                      APLOGNO(02919) 
+        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
                       "h2_session:  stream(%ld-%d): on_data_chunk for unknown stream",
                       session->id, (int)stream_id);
         rv = nghttp2_submit_rst_stream(ngh2, NGHTTP2_FLAG_NONE, stream_id,
@@ -165,11 +223,11 @@ static int on_data_chunk_recv_cb(nghttp2
     
     status = h2_stream_write_data(stream, (const char *)data, len);
     ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, session->c,
-                  "h2_stream(%ld-%d): written DATA, length %d",
-                  session->id, stream_id, (int)len);
+                  "h2_stream(%ld-%d): data_chunk_recv, written %ld bytes",
+                  session->id, stream_id, (long)len);
     if (status != APR_SUCCESS) {
         rv = nghttp2_submit_rst_stream(ngh2, NGHTTP2_FLAG_NONE, stream_id,
-                                       NGHTTP2_INTERNAL_ERROR);
+                                       H2_STREAM_RST(stream, H2_ERR_INTERNAL_ERROR));
         if (nghttp2_is_fatal(rv)) {
             return NGHTTP2_ERR_CALLBACK_FAILURE;
         }
@@ -177,56 +235,7 @@ static int on_data_chunk_recv_cb(nghttp2
     return 0;
 }
 
-static int before_frame_send_cb(nghttp2_session *ngh2,
-                                const nghttp2_frame *frame,
-                                void *userp)
-{
-    h2_session *session = (h2_session *)userp;
-    (void)ngh2;
-
-    if (session->aborted) {
-        return NGHTTP2_ERR_CALLBACK_FAILURE;
-    }
-    if (APLOGctrace2(session->c)) {
-        char buffer[256];
-        frame_print(frame, buffer, sizeof(buffer)/sizeof(buffer[0]));
-        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
-                      "h2_session(%ld): before_frame_send %s", 
-                      session->id, buffer);
-    }
-    return 0;
-}
-
-static int on_frame_send_cb(nghttp2_session *ngh2,
-                            const nghttp2_frame *frame,
-                            void *userp)
-{
-    h2_session *session = (h2_session *)userp;
-    (void)ngh2; (void)frame;
-    ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, session->c,
-                  "h2_session(%ld): on_frame_send", session->id);
-    return 0;
-}
-
-static int on_frame_not_send_cb(nghttp2_session *ngh2,
-                                const nghttp2_frame *frame,
-                                int lib_error_code, void *userp)
-{
-    h2_session *session = (h2_session *)userp;
-    (void)ngh2;
-    
-    if (APLOGctrace2(session->c)) {
-        char buffer[256];
-        
-        frame_print(frame, buffer, sizeof(buffer)/sizeof(buffer[0]));
-        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
-                      "h2_session: callback on_frame_not_send error=%d %s",
-                      lib_error_code, buffer);
-    }
-    return 0;
-}
-
-static apr_status_t stream_destroy(h2_session *session, 
+static apr_status_t stream_release(h2_session *session, 
                                    h2_stream *stream,
                                    uint32_t error_code) 
 {
@@ -242,11 +251,13 @@ static apr_status_t stream_destroy(h2_se
         ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
                       "h2_stream(%ld-%d): closing with err=%d %s", 
                       session->id, (int)stream->id, (int)error_code,
-                      nghttp2_strerror(error_code));
+                      h2_h2_err_description(error_code));
+        h2_stream_rst(stream, error_code);
     }
     
-    h2_stream_set_remove(session->streams, stream);
-    return h2_mplx_cleanup_stream(session->mplx, stream);
+    return h2_conn_io_writeb(&session->io,
+                             h2_bucket_eos_create(session->c->bucket_alloc, 
+                                                  stream));
 }
 
 static int on_stream_close_cb(nghttp2_session *ngh2, int32_t stream_id,
@@ -259,33 +270,30 @@ static int on_stream_close_cb(nghttp2_se
     if (session->aborted) {
         return NGHTTP2_ERR_CALLBACK_FAILURE;
     }
-    stream = h2_stream_set_get(session->streams, stream_id);
+    stream = h2_session_get_stream(session, stream_id);
     if (stream) {
-        stream_destroy(session, stream, error_code);
+        stream_release(session, stream, error_code);
     }
-    
-    if (error_code) {
-        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
-                      "h2_stream(%ld-%d): close error %d",
-                      session->id, (int)stream_id, error_code);
-    }
-    
     return 0;
 }
 
 static int on_begin_headers_cb(nghttp2_session *ngh2,
                                const nghttp2_frame *frame, void *userp)
 {
-    /* This starts a new stream. */
-    int rv;
+    h2_session *session = (h2_session *)userp;
+    h2_stream *s;
+    
+    /* We may see HEADERs at the start of a stream or after all DATA
+     * streams to carry trailers. */
     (void)ngh2;
-    rv = stream_open((h2_session *)userp, frame->hd.stream_id);
-    if (rv != NGHTTP2_ERR_CALLBACK_FAILURE) {
-      /* on_header_cb or on_frame_recv_cb will dectect that stream
-         does not exist and submit RST_STREAM. */
-      return 0;
+    s = h2_session_get_stream(session, frame->hd.stream_id);
+    if (s) {
+        /* nop */
+    }
+    else {
+        s = h2_session_open_stream((h2_session *)userp, frame->hd.stream_id);
     }
-    return NGHTTP2_ERR_CALLBACK_FAILURE;
+    return s? 0 : NGHTTP2_ERR_CALLBACK_FAILURE;
 }
 
 static int on_header_cb(nghttp2_session *ngh2, const nghttp2_frame *frame,
@@ -303,8 +311,8 @@ static int on_header_cb(nghttp2_session
     if (session->aborted) {
         return NGHTTP2_ERR_CALLBACK_FAILURE;
     }
-    stream = h2_stream_set_get(session->streams,
-                                           frame->hd.stream_id);
+    
+    stream = h2_session_get_stream(session, frame->hd.stream_id);
     if (!stream) {
         ap_log_cerror(APLOG_MARK, APLOG_ERR, 0, session->c,
                       APLOGNO(02920) 
@@ -313,9 +321,9 @@ static int on_header_cb(nghttp2_session
         return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
     }
     
-    status = h2_stream_write_header(stream,
-                                               (const char *)name, namelen,
-                                               (const char *)value, valuelen);
+    status = h2_stream_add_header(stream, (const char *)name, namelen,
+                                  (const char *)value, valuelen);
+                                    
     if (status != APR_SUCCESS) {
         return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
     }
@@ -331,63 +339,67 @@ static int on_frame_recv_cb(nghttp2_sess
                             const nghttp2_frame *frame,
                             void *userp)
 {
-    int rv;
     h2_session *session = (h2_session *)userp;
     apr_status_t status = APR_SUCCESS;
+    h2_stream *stream;
+    
     if (session->aborted) {
         return NGHTTP2_ERR_CALLBACK_FAILURE;
     }
     
-    ++session->frames_received;
-    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c,
-                  "h2_session(%ld): on_frame_rcv #%ld, type=%d", session->id,
+    ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, session->c,
+                  "h2_stream(%ld-%d): on_frame_rcv #%ld, type=%d", 
+                  session->id, frame->hd.stream_id, 
                   (long)session->frames_received, frame->hd.type);
+
+    ++session->frames_received;
     switch (frame->hd.type) {
-        case NGHTTP2_HEADERS: {
-            int eos;
-            h2_stream * stream = h2_stream_set_get(session->streams,
-                                                   frame->hd.stream_id);
-            if (stream == NULL) {
-                ap_log_cerror(APLOG_MARK, APLOG_ERR, 0, session->c,
-                              APLOGNO(02921) 
-                              "h2_session:  stream(%ld-%d): HEADERS frame "
-                              "for unknown stream", session->id,
-                              (int)frame->hd.stream_id);
-                rv = nghttp2_submit_rst_stream(ng2s, NGHTTP2_FLAG_NONE,
-                                               frame->hd.stream_id,
-                                               NGHTTP2_INTERNAL_ERROR);
-                if (nghttp2_is_fatal(rv)) {
-                    return NGHTTP2_ERR_CALLBACK_FAILURE;
+        case NGHTTP2_HEADERS:
+            /* This can be HEADERS for a new stream, defining the request,
+             * or HEADER may come after DATA at the end of a stream as in
+             * trailers */
+            stream = h2_session_get_stream(session, frame->hd.stream_id);
+            if (stream) {
+                int eos = (frame->hd.flags & NGHTTP2_FLAG_END_STREAM);
+                
+                if (h2_stream_is_scheduled(stream)) {
+                    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c,
+                                  "h2_stream(%ld-%d): TRAILER, eos=%d", 
+                                  session->id, frame->hd.stream_id, eos);
+                    if (eos) {
+                        status = h2_stream_close_input(stream);
+                    }
+                }
+                else {
+                    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c,
+                                  "h2_stream(%ld-%d): HEADER, eos=%d", 
+                                  session->id, frame->hd.stream_id, eos);
+                    status = stream_schedule(session, stream, eos);
                 }
-                return 0;
             }
-
-            eos = (frame->hd.flags & NGHTTP2_FLAG_END_STREAM);
-            status = stream_end_headers(session, stream, eos);
-
+            else {
+                status = APR_EINVAL;
+            }
             break;
-        }
-        case NGHTTP2_DATA: {
-            h2_stream * stream = h2_stream_set_get(session->streams,
-                                                   frame->hd.stream_id);
-            if (stream == NULL) {
-                ap_log_cerror(APLOG_MARK, APLOG_ERR, 0, session->c,
-                              APLOGNO(02922) 
-                              "h2_session:  stream(%ld-%d): DATA frame "
-                              "for unknown stream", session->id,
-                              (int)frame->hd.stream_id);
-                rv = nghttp2_submit_rst_stream(ng2s, NGHTTP2_FLAG_NONE,
-                                               frame->hd.stream_id,
-                                               NGHTTP2_INTERNAL_ERROR);
-                if (nghttp2_is_fatal(rv)) {
-                    return NGHTTP2_ERR_CALLBACK_FAILURE;
+        case NGHTTP2_DATA:
+            stream = h2_session_get_stream(session, frame->hd.stream_id);
+            if (stream) {
+                int eos = (frame->hd.flags & NGHTTP2_FLAG_END_STREAM);
+                ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c,
+                              "h2_stream(%ld-%d): DATA, len=%ld, eos=%d", 
+                              session->id, frame->hd.stream_id, 
+                              (long)frame->hd.length, eos);
+                if (eos) {
+                    status = h2_stream_close_input(stream);
                 }
-                return 0;
+            }
+            else {
+                status = APR_EINVAL;
             }
             break;
-        }
-        case NGHTTP2_PRIORITY: {
-            ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c,
+        case NGHTTP2_PRIORITY:
+            session->reprioritize = 1;
+            ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, session->c,
                           "h2_session:  stream(%ld-%d): PRIORITY frame "
                           " weight=%d, dependsOn=%d, exclusive=%d", 
                           session->id, (int)frame->hd.stream_id,
@@ -395,7 +407,13 @@ static int on_frame_recv_cb(nghttp2_sess
                           frame->priority.pri_spec.stream_id,
                           frame->priority.pri_spec.exclusive);
             break;
-        }
+        case NGHTTP2_WINDOW_UPDATE:
+            ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, session->c,
+                          "h2_session:  stream(%ld-%d): WINDOW_UPDATE "
+                          "incr=%d", 
+                          session->id, (int)frame->hd.stream_id,
+                          frame->window_update.window_size_increment);
+            break;
         default:
             if (APLOGctrace2(session->c)) {
                 char buffer[256];
@@ -408,23 +426,10 @@ static int on_frame_recv_cb(nghttp2_sess
             break;
     }
 
-    /* only DATA and HEADERS frame can bear END_STREAM flag.  Other
-       frame types may have other flag which has the same value, so we
-       have to check the frame type first.  */
-    if ((frame->hd.type == NGHTTP2_DATA || frame->hd.type == NGHTTP2_HEADERS) &&
-        frame->hd.flags & NGHTTP2_FLAG_END_STREAM) {
-        h2_stream * stream = h2_stream_set_get(session->streams,
-                                               frame->hd.stream_id);
-        if (stream != NULL) {
-            status = h2_stream_write_eos(stream);
-            ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, session->c,
-                          "h2_stream(%ld-%d): input closed",
-                          session->id, (int)frame->hd.stream_id);
-        }
-    }
-    
     if (status != APR_SUCCESS) {
-        ap_log_cerror(APLOG_MARK, APLOG_ERR, status, session->c,
+        int rv;
+        
+        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, session->c,
                       APLOGNO(02923) 
                       "h2_session: stream(%ld-%d): error handling frame",
                       session->id, (int)frame->hd.stream_id);
@@ -434,24 +439,20 @@ static int on_frame_recv_cb(nghttp2_sess
         if (nghttp2_is_fatal(rv)) {
             return NGHTTP2_ERR_CALLBACK_FAILURE;
         }
-        return 0;
     }
     
     return 0;
 }
 
-static apr_status_t send_data(h2_session *session, const char *data, 
-                              apr_size_t length)
-{
-    return h2_conn_io_write(&session->io, data, length);
-}
-
 static apr_status_t pass_data(void *ctx, 
-                              const char *data, apr_size_t length)
+                              const char *data, apr_off_t length)
 {
-    return send_data((h2_session*)ctx, data, length);
+    return h2_conn_io_write(&((h2_session*)ctx)->io, data, length);
 }
 
+
+static char immortal_zeros[H2_MAX_PADLEN];
+
 static int on_send_data_cb(nghttp2_session *ngh2, 
                            nghttp2_frame *frame, 
                            const uint8_t *framehd, 
@@ -462,7 +463,7 @@ static int on_send_data_cb(nghttp2_sessi
     apr_status_t status = APR_SUCCESS;
     h2_session *session = (h2_session *)userp;
     int stream_id = (int)frame->hd.stream_id;
-    const unsigned char padlen = frame->data.padlen;
+    unsigned char padlen;
     int eos;
     h2_stream *stream;
     
@@ -472,7 +473,12 @@ static int on_send_data_cb(nghttp2_sessi
         return NGHTTP2_ERR_CALLBACK_FAILURE;
     }
     
-    stream = h2_stream_set_get(session->streams, stream_id);
+    if (frame->data.padlen > H2_MAX_PADLEN) {
+        return NGHTTP2_ERR_PROTO;
+    }
+    padlen = (unsigned char)frame->data.padlen;
+    
+    stream = h2_session_get_stream(session, stream_id);
     if (!stream) {
         ap_log_cerror(APLOG_MARK, APLOG_ERR, APR_NOTFOUND, session->c,
                       APLOGNO(02924) 
@@ -481,45 +487,74 @@ static int on_send_data_cb(nghttp2_sessi
         return NGHTTP2_ERR_CALLBACK_FAILURE;
     }
     
-    status = send_data(session, (const char *)framehd, 9);
-    if (status == APR_SUCCESS) {
+    ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, session->c,
+                  "h2_stream(%ld-%d): send_data_cb for %ld bytes",
+                  session->id, (int)stream_id, (long)length);
+                  
+    if (h2_conn_io_is_buffered(&session->io)) {
+        status = h2_conn_io_write(&session->io, (const char *)framehd, 9);
+        if (status == APR_SUCCESS) {
+            if (padlen) {
+                status = h2_conn_io_write(&session->io, (const char *)&padlen, 1);
+            }
+            
+            if (status == APR_SUCCESS) {
+                apr_off_t len = length;
+                status = h2_stream_readx(stream, pass_data, session, &len, &eos);
+                if (status == APR_SUCCESS && len != length) {
+                    status = APR_EINVAL;
+                }
+            }
+            
+            if (status == APR_SUCCESS && padlen) {
+                if (padlen) {
+                    status = h2_conn_io_write(&session->io, immortal_zeros, padlen);
+                }
+            }
+        }
+    }
+    else {
+        apr_bucket *b;
+        char *header = apr_pcalloc(stream->pool, 10);
+        memcpy(header, (const char *)framehd, 9);
         if (padlen) {
-            status = send_data(session, (const char *)&padlen, 1);
+            header[9] = (char)padlen;
         }
-
+        b = apr_bucket_pool_create(header, padlen? 10 : 9, 
+                                   stream->pool, session->c->bucket_alloc);
+        status = h2_conn_io_writeb(&session->io, b);
+        
         if (status == APR_SUCCESS) {
-            apr_size_t len = length;
-            status = h2_stream_readx(stream, pass_data, session, 
-                                     &len, &eos);
+            apr_off_t len = length;
+            status = h2_stream_read_to(stream, session->io.output, &len, &eos);
             if (status == APR_SUCCESS && len != length) {
                 status = APR_EINVAL;
             }
         }
-        
+            
         if (status == APR_SUCCESS && padlen) {
-            if (padlen) {
-                char pad[256];
-                memset(pad, 0, padlen);
-                status = send_data(session, pad, padlen);
-            }
+            b = apr_bucket_immortal_create(immortal_zeros, padlen, 
+                                           session->c->bucket_alloc);
+            status = h2_conn_io_writeb(&session->io, b);
         }
     }
     
+    
     if (status == APR_SUCCESS) {
+        stream->data_frames_sent++;
+        h2_conn_io_consider_flush(&session->io);
         return 0;
     }
-    else if (status != APR_EOF) {
-        ap_log_cerror(APLOG_MARK, APLOG_ERR, status, session->c,
+    else {
+        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, session->c,
                       APLOGNO(02925) 
                       "h2_stream(%ld-%d): failed send_data_cb",
                       session->id, (int)stream_id);
-        return NGHTTP2_ERR_CALLBACK_FAILURE;
     }
     
     return h2_session_status_from_apr_status(status);
 }
 
-
 #define NGH2_SET_CALLBACK(callbacks, name, fn)\
 nghttp2_session_callbacks_set_##name##_callback(callbacks, fn)
 
@@ -537,9 +572,6 @@ static apr_status_t init_callbacks(conn_
     NGH2_SET_CALLBACK(*pcb, on_frame_recv, on_frame_recv_cb);
     NGH2_SET_CALLBACK(*pcb, on_invalid_frame_recv, on_invalid_frame_recv_cb);
     NGH2_SET_CALLBACK(*pcb, on_data_chunk_recv, on_data_chunk_recv_cb);
-    NGH2_SET_CALLBACK(*pcb, before_frame_send, before_frame_send_cb);
-    NGH2_SET_CALLBACK(*pcb, on_frame_send, on_frame_send_cb);
-    NGH2_SET_CALLBACK(*pcb, on_frame_not_send, on_frame_not_send_cb);
     NGH2_SET_CALLBACK(*pcb, on_stream_close, on_stream_close_cb);
     NGH2_SET_CALLBACK(*pcb, on_begin_headers, on_begin_headers_cb);
     NGH2_SET_CALLBACK(*pcb, on_header, on_header_cb);
@@ -548,6 +580,16 @@ static apr_status_t init_callbacks(conn_
     return APR_SUCCESS;
 }
 
+static apr_status_t session_pool_cleanup(void *data)
+{
+    h2_session *session = data;
+    
+    /* keep us from destroying the pool, since that is already ongoing. */
+    session->pool = NULL;
+    h2_session_destroy(session);
+    return APR_SUCCESS;
+}
+
 static h2_session *h2_session_create_int(conn_rec *c,
                                          request_rec *r,
                                          h2_config *config, 
@@ -570,6 +612,8 @@ static h2_session *h2_session_create_int
         session->c = c;
         session->r = r;
         
+        apr_pool_pre_cleanup_register(pool, session, session_pool_cleanup);
+        
         session->max_stream_count = h2_config_geti(config, H2_CONF_MAX_STREAMS);
         session->max_stream_mem = h2_config_geti(config, H2_CONF_STREAM_MAX_MEM);
 
@@ -580,7 +624,7 @@ static h2_session *h2_session_create_int
             return NULL;
         }
         
-        session->streams = h2_stream_set_create(session->pool);
+        session->streams = h2_stream_set_create(session->pool, session->max_stream_count);
         
         session->workers = workers;
         session->mplx = h2_mplx_create(c, session->pool, workers);
@@ -641,15 +685,37 @@ h2_session *h2_session_rcreate(request_r
     return h2_session_create_int(r->connection, r, config, workers);
 }
 
-void h2_session_destroy(h2_session *session)
+static void h2_session_cleanup(h2_session *session)
 {
     AP_DEBUG_ASSERT(session);
+    /* This is an early cleanup of the session that may
+     * discard what is no longer necessary for *new* streams
+     * and general HTTP/2 processing.
+     * At this point, all frames are in transit or somehwere in
+     * our buffers or passed down output filters.
+     * h2 streams might still being written out.
+     */
+    if (session->ngh2) {
+        nghttp2_session_del(session->ngh2);
+        session->ngh2 = NULL;
+    }
+    if (session->spare) {
+        apr_pool_destroy(session->spare);
+        session->spare = NULL;
+    }
     if (session->mplx) {
         h2_mplx_release_and_join(session->mplx, session->iowait);
         session->mplx = NULL;
     }
+}
+
+void h2_session_destroy(h2_session *session)
+{
+    AP_DEBUG_ASSERT(session);
+    h2_session_cleanup(session);
+    
     if (session->streams) {
-        if (h2_stream_set_size(session->streams)) {
+        if (!h2_stream_set_is_empty(session->streams)) {
             ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, session->c,
                           "h2_session(%ld): destroy, %d streams open",
                           session->id, (int)h2_stream_set_size(session->streams));
@@ -657,35 +723,37 @@ void h2_session_destroy(h2_session *sess
         h2_stream_set_destroy(session->streams);
         session->streams = NULL;
     }
-    if (session->ngh2) {
-        nghttp2_session_del(session->ngh2);
-        session->ngh2 = NULL;
-    }
-    h2_conn_io_destroy(&session->io);
-    
-    if (session->iowait) {
-        apr_thread_cond_destroy(session->iowait);
-        session->iowait = NULL;
-    }
-    
     if (session->pool) {
         apr_pool_destroy(session->pool);
     }
 }
 
+
+void h2_session_eoc_callback(h2_session *session)
+{
+    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c,
+                  "session(%ld): cleanup and destroy", session->id);
+    apr_pool_cleanup_kill(session->pool, session, session_pool_cleanup);
+    h2_session_destroy(session);
+}
+
 static apr_status_t h2_session_abort_int(h2_session *session, int reason)
 {
     AP_DEBUG_ASSERT(session);
     if (!session->aborted) {
         session->aborted = 1;
-        if (session->ngh2) {
-            
-            if (!reason) {
+        
+        if (session->ngh2) {            
+            if (NGHTTP2_ERR_EOF == reason) {
+                /* This is our way of indication that the connection is
+                 * gone. No use to send any GOAWAY frames. */
+                nghttp2_session_terminate_session(session->ngh2, reason);
+            }
+            else if (!reason) {
                 nghttp2_submit_goaway(session->ngh2, NGHTTP2_FLAG_NONE, 
                                       session->max_stream_received, 
                                       reason, NULL, 0);
                 nghttp2_session_send(session->ngh2);
-                h2_conn_io_flush(&session->io);
             }
             else {
                 const char *err = nghttp2_strerror(reason);
@@ -694,21 +762,13 @@ static apr_status_t h2_session_abort_int
                               "session(%ld): aborting session, reason=%d %s",
                               session->id, reason, err);
                 
-                if (NGHTTP2_ERR_EOF == reason) {
-                    /* This is our way of indication that the connection is
-                     * gone. No use to send any GOAWAY frames. */
-                    nghttp2_session_terminate_session(session->ngh2, reason);
-                }
-                else {
-                    /* The connection might still be there and we shut down
-                     * with GOAWAY and reason information. */
-                     nghttp2_submit_goaway(session->ngh2, NGHTTP2_FLAG_NONE, 
-                                           session->max_stream_received, 
-                                           reason, (const uint8_t *)err, 
-                                           strlen(err));
-                     nghttp2_session_send(session->ngh2);
-                     h2_conn_io_flush(&session->io);
-                }
+                /* The connection might still be there and we shut down
+                 * with GOAWAY and reason information. */
+                nghttp2_submit_goaway(session->ngh2, NGHTTP2_FLAG_NONE, 
+                                      session->max_stream_received, 
+                                      reason, (const uint8_t *)err, 
+                                      strlen(err));
+                nghttp2_session_send(session->ngh2);
             }
         }
         h2_mplx_abort(session->mplx);
@@ -731,10 +791,12 @@ apr_status_t h2_session_abort(h2_session
                 rv = 0;                            /* ...gracefully shut down */
                 break;
             case APR_EBADF:        /* connection unusable, terminate silently */
-            case APR_ECONNABORTED:
-                rv = NGHTTP2_ERR_EOF;
-                break;
             default:
+                if (APR_STATUS_IS_ECONNABORTED(reason)
+                    || APR_STATUS_IS_ECONNRESET(reason)
+                    || APR_STATUS_IS_EBADF(reason)) {
+                    rv = NGHTTP2_ERR_EOF;
+                }
                 break;
         }
     }
@@ -746,6 +808,8 @@ apr_status_t h2_session_start(h2_session
     apr_status_t status = APR_SUCCESS;
     h2_config *config;
     nghttp2_settings_entry settings[3];
+    size_t slen;
+    int i;
     
     AP_DEBUG_ASSERT(session);
     /* Start the conversation by submitting our SETTINGS frame */
@@ -789,8 +853,8 @@ apr_status_t h2_session_start(h2_session
         }
         
         /* Now we need to auto-open stream 1 for the request we got. */
-        *rv = stream_open(session, 1);
-        if (*rv != 0) {
+        stream = h2_session_open_stream(session, 1);
+        if (!stream) {
             status = APR_EGENERAL;
             ap_log_rerror(APLOG_MARK, APLOG_ERR, status, session->r,
                           APLOGNO(02933) "open stream 1: %s", 
@@ -798,34 +862,29 @@ apr_status_t h2_session_start(h2_session
             return status;
         }
         
-        stream = h2_stream_set_get(session->streams, 1);
-        if (stream == NULL) {
-            status = APR_EGENERAL;
-            ap_log_rerror(APLOG_MARK, APLOG_ERR, status, session->r,
-                          APLOGNO(02934) "lookup of stream 1");
-            return status;
-        }
-        
-        status = h2_stream_rwrite(stream, session->r);
+        status = h2_stream_set_request(stream, session->r);
         if (status != APR_SUCCESS) {
             return status;
         }
-        status = stream_end_headers(session, stream, 1);
+        status = stream_schedule(session, stream, 1);
         if (status != APR_SUCCESS) {
             return status;
         }
     }
 
-    settings[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
-    settings[0].value = (uint32_t)session->max_stream_count;
-    settings[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
-    settings[1].value = h2_config_geti(config, H2_CONF_WIN_SIZE);
-    settings[2].settings_id = NGHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE;
-    settings[2].value = 64*1024;
+    slen = 0;
+    settings[slen].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
+    settings[slen].value = (uint32_t)session->max_stream_count;
+    ++slen;
+    i = h2_config_geti(config, H2_CONF_WIN_SIZE);
+    if (i != H2_INITIAL_WINDOW_SIZE) {
+        settings[slen].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
+        settings[slen].value = i;
+        ++slen;
+    }
     
     *rv = nghttp2_submit_settings(session->ngh2, NGHTTP2_FLAG_NONE,
-                                 settings,
-                                 sizeof(settings)/sizeof(settings[0]));
+                                  settings, slen);
     if (*rv != 0) {
         status = APR_EGENERAL;
         ap_log_cerror(APLOG_MARK, APLOG_ERR, status, session->c,
@@ -836,11 +895,6 @@ apr_status_t h2_session_start(h2_session
     return status;
 }
 
-static int h2_session_want_write(h2_session *session)
-{
-    return nghttp2_session_want_write(session->ngh2);
-}
-
 typedef struct {
     h2_session *session;
     int resume_count;
@@ -853,7 +907,7 @@ static int resume_on_data(void *ctx, h2_
     AP_DEBUG_ASSERT(stream);
     
     if (h2_stream_is_suspended(stream)) {
-        if (h2_mplx_out_has_data_for(stream->m, stream->id)) {
+        if (h2_mplx_out_has_data_for(stream->session->mplx, stream->id)) {
             int rv;
             h2_stream_set_suspended(stream, 0);
             ++rctx->resume_count;
@@ -886,98 +940,18 @@ static int h2_session_resume_streams_wit
     return 0;
 }
 
-static void update_window(void *ctx, int stream_id, apr_size_t bytes_read)
+static void update_window(void *ctx, int stream_id, apr_off_t bytes_read)
 {
     h2_session *session = (h2_session*)ctx;
     nghttp2_session_consume(session->ngh2, stream_id, bytes_read);
 }
 
-static apr_status_t h2_session_update_windows(h2_session *session)
-{
-    return h2_mplx_in_update_windows(session->mplx, update_window, session);
-}
-
-apr_status_t h2_session_write(h2_session *session, apr_interval_time_t timeout)
-{
-    apr_status_t status = APR_EAGAIN;
-    h2_stream *stream = NULL;
-    int flush_output = 0;
-    
-    AP_DEBUG_ASSERT(session);
-    
-    /* Check that any pending window updates are sent. */
-    status = h2_session_update_windows(session);
-    if (status == APR_SUCCESS) {
-        flush_output = 1;
-    }
-    else if (status != APR_EAGAIN) {
-        return status;
-    }
-    
-    if (h2_session_want_write(session)) {
-        int rv;
-        status = APR_SUCCESS;
-        rv = nghttp2_session_send(session->ngh2);
-        if (rv != 0) {
-            ap_log_cerror( APLOG_MARK, APLOG_DEBUG, 0, session->c,
-                          "h2_session: send: %s", nghttp2_strerror(rv));
-            if (nghttp2_is_fatal(rv)) {
-                h2_session_abort_int(session, rv);
-                status = APR_ECONNABORTED;
-            }
-        }
-        flush_output = 1;
-    }
-    
-    /* If we have responses ready, submit them now. */
-    while ((stream = h2_mplx_next_submit(session->mplx, 
-                                         session->streams)) != NULL) {
-        status = h2_session_handle_response(session, stream);
-        flush_output = 1;
-    }
-    
-    if (h2_session_resume_streams_with_data(session) > 0) {
-        flush_output = 1;
-    }
-    
-    if (!flush_output && timeout > 0 && !h2_session_want_write(session)) {
-        status = h2_mplx_out_trywait(session->mplx, timeout, session->iowait);
-
-        if (status != APR_TIMEUP
-            && h2_session_resume_streams_with_data(session) > 0) {
-            flush_output = 1;
-        }
-        else {
-            /* nothing happened to ongoing streams, do some house-keeping */
-        }
-    }
-    
-    if (h2_session_want_write(session)) {
-        int rv;
-        status = APR_SUCCESS;
-        rv = nghttp2_session_send(session->ngh2);
-        if (rv != 0) {
-            ap_log_cerror( APLOG_MARK, APLOG_DEBUG, 0, session->c,
-                          "h2_session: send2: %s", nghttp2_strerror(rv));
-            if (nghttp2_is_fatal(rv)) {
-                h2_session_abort_int(session, rv);
-                status = APR_ECONNABORTED;
-            }
-        }
-        flush_output = 1;
-    }
-    
-    if (flush_output) {
-        h2_conn_io_flush(&session->io);
-    }
-    
-    return status;
-}
-
 h2_stream *h2_session_get_stream(h2_session *session, int stream_id)
 {
-    AP_DEBUG_ASSERT(session);
-    return h2_stream_set_get(session->streams, stream_id);
+    if (!session->last_stream || stream_id != session->last_stream->id) {
+        session->last_stream = h2_stream_set_get(session->streams, stream_id);
+    }
+    return session->last_stream;
 }
 
 /* h2_io_on_read_cb implementation that offers the data read
@@ -1010,20 +984,21 @@ static apr_status_t session_receive(cons
     return APR_SUCCESS;
 }
 
-apr_status_t h2_session_read(h2_session *session, apr_read_type_e block)
-{
-    AP_DEBUG_ASSERT(session);
-    return h2_conn_io_read(&session->io, block, session_receive, session);
-}
-
 apr_status_t h2_session_close(h2_session *session)
 {
     AP_DEBUG_ASSERT(session);
-    return session->aborted? APR_SUCCESS : h2_conn_io_flush(&session->io);
+    if (!session->aborted) {
+        h2_session_abort_int(session, 0);
+    }
+    ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0,session->c,
+                  "h2_session: closing, writing eoc");
+    
+    h2_session_cleanup(session);              
+    return h2_conn_io_writeb(&session->io,
+                             h2_bucket_eoc_create(session->c->bucket_alloc, 
+                                                  session));
 }
 
-/* The session wants to send more DATA for the given stream.
- */
 static ssize_t stream_data_cb(nghttp2_session *ng2s,
                               int32_t stream_id,
                               uint8_t *buf,
@@ -1033,18 +1008,27 @@ static ssize_t stream_data_cb(nghttp2_se
                               void *puser)
 {
     h2_session *session = (h2_session *)puser;
-    apr_size_t nread = length;
+    apr_off_t nread = length;
     int eos = 0;
     apr_status_t status;
     h2_stream *stream;
     AP_DEBUG_ASSERT(session);
     
+    /* The session wants to send more DATA for the stream. We need
+     * to find out how much of the requested length we can send without
+     * blocking.
+     * Indicate EOS when we encounter it or DEFERRED if the stream
+     * should be suspended.
+     * TODO: for handling of TRAILERS,  the EOF indication needs
+     * to be aware of that.
+     */
+ 
     (void)ng2s;
     (void)buf;
     (void)source;
-    stream = h2_stream_set_get(session->streams, stream_id);
+    stream = h2_session_get_stream(session, stream_id);
     if (!stream) {
-        ap_log_cerror(APLOG_MARK, APLOG_ERR, APR_NOTFOUND, session->c,
+        ap_log_cerror(APLOG_MARK, APLOG_ERR, 0, session->c,
                       APLOGNO(02937) 
                       "h2_stream(%ld-%d): data requested but stream not found",
                       session->id, (int)stream_id);
@@ -1062,6 +1046,10 @@ static ssize_t stream_data_cb(nghttp2_se
         case APR_SUCCESS:
             break;
             
+        case APR_ECONNRESET:
+            return nghttp2_submit_rst_stream(ng2s, NGHTTP2_FLAG_NONE,
+                stream->id, stream->rst_error);
+            
         case APR_EAGAIN:
             /* If there is no data available, our session will automatically
              * suspend this stream and not ask for more data until we resume
@@ -1088,6 +1076,18 @@ static ssize_t stream_data_cb(nghttp2_se
     }
     
     if (eos) {
+        apr_table_t *trailers = h2_stream_get_trailers(stream);
+        if (trailers && !apr_is_empty_table(trailers)) {
+            h2_ngheader *nh;
+            int rv;
+            
+            nh = h2_util_ngheader_make(stream->pool, trailers);
+            rv = nghttp2_submit_trailer(ng2s, stream->id, nh->nv, nh->nvlen);
+            if (rv < 0) {
+                nread = rv;
+            }
+        }
+        
         *data_flags |= NGHTTP2_DATA_FLAG_EOF;
     }
     
@@ -1100,57 +1100,75 @@ typedef struct {
     size_t offset;
 } nvctx_t;
 
-static int submit_response(h2_session *session, h2_response *response)
-{
-    nghttp2_data_provider provider;
-    int rv;
-    
-    memset(&provider, 0, sizeof(provider));
-    provider.source.fd = response->stream_id;
-    provider.read_callback = stream_data_cb;
-    
-    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c,
-                  "h2_stream(%ld-%d): submitting response %s",
-                  session->id, response->stream_id, response->status);
-    
-    rv = nghttp2_submit_response(session->ngh2, response->stream_id,
-                                 response->ngheader->nv, 
-                                 response->ngheader->nvlen, &provider);
-    
-    if (rv != 0) {
-        ap_log_cerror(APLOG_MARK, APLOG_ERR, 0, session->c,
-                      APLOGNO(02939) "h2_stream(%ld-%d): submit_response: %s",
-                      session->id, response->stream_id, nghttp2_strerror(rv));
-    }
-    else {
-        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
-                      "h2_stream(%ld-%d): submitted response %s, rv=%d",
-                      session->id, response->stream_id, 
-                      response->status, rv);
-    }
-    return rv;
-}
-
-/* Start submitting the response to a stream request. This is possible
+/**
+ * Start submitting the response to a stream request. This is possible
  * once we have all the response headers. The response body will be
  * read by the session using the callback we supply.
  */
-apr_status_t h2_session_handle_response(h2_session *session, h2_stream *stream)
+static apr_status_t submit_response(h2_session *session, h2_stream *stream)
 {
     apr_status_t status = APR_SUCCESS;
     int rv = 0;
     AP_DEBUG_ASSERT(session);
     AP_DEBUG_ASSERT(stream);
-    AP_DEBUG_ASSERT(stream->response);
+    AP_DEBUG_ASSERT(stream->response || stream->rst_error);
     
-    if (stream->response->ngheader) {
-        rv = submit_response(session, stream->response);
+    if (stream->submitted) {
+        rv = NGHTTP2_PROTOCOL_ERROR;
+    }
+    else if (stream->response && stream->response->header) {
+        nghttp2_data_provider provider;
+        h2_response *response = stream->response;
+        h2_ngheader *ngh;
+        
+        memset(&provider, 0, sizeof(provider));
+        provider.source.fd = stream->id;
+        provider.read_callback = stream_data_cb;
+        
+        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
+                      "h2_stream(%ld-%d): submit response %d",
+                      session->id, stream->id, response->http_status);
+        
+        ngh = h2_util_ngheader_make_res(stream->pool, response->http_status, 
+                                        response->header);
+        rv = nghttp2_submit_response(session->ngh2, response->stream_id,
+                                     ngh->nv, ngh->nvlen, &provider);
+        
+        /* If the submit worked,
+         * and this stream is not a pushed one itself,
+         * and HTTP/2 server push is enabled here,
+         * and the response is in the range 200-299 *),
+         * and the remote side has pushing enabled,
+         * -> find and perform any pushes on this stream
+         * 
+         * *) the response code is relevant, as we do not want to 
+         *    make pushes on 401 or 403 codes, neiterh on 301/302
+         *    and friends. And if we see a 304, we do not push either
+         *    as the client, having this resource in its cache, might
+         *    also have the pushed ones as well.
+         */
+        if (!rv 
+            && !stream->initiated_on
+            && h2_config_geti(h2_config_get(session->c), H2_CONF_PUSH)
+            && H2_HTTP_2XX(response->http_status)
+            && h2_session_push_enabled(session)) {
+            
+            h2_stream_submit_pushes(stream);
+        }
     }
     else {
+        int err = H2_STREAM_RST(stream, H2_ERR_PROTOCOL_ERROR);
+        
+        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
+                      "h2_stream(%ld-%d): RST_STREAM, err=%d",
+                      session->id, stream->id, err);
+
         rv = nghttp2_submit_rst_stream(session->ngh2, NGHTTP2_FLAG_NONE,
-                                       stream->id, NGHTTP2_PROTOCOL_ERROR);
+                                       stream->id, err);
     }
     
+    stream->submitted = 1;
+
     if (nghttp2_is_fatal(rv)) {
         status = APR_EGENERAL;
         h2_session_abort_int(session, rv);
@@ -1158,37 +1176,88 @@ apr_status_t h2_session_handle_response(
                       APLOGNO(02940) "submit_response: %s", 
                       nghttp2_strerror(rv));
     }
+    
     return status;
 }
 
-int h2_session_is_done(h2_session *session)
+struct h2_stream *h2_session_push(h2_session *session, h2_stream *is,
+                                  h2_push *push)
 {
-    AP_DEBUG_ASSERT(session);
-    return (session->aborted
-            || !session->ngh2
-            || (!nghttp2_session_want_read(session->ngh2)
-                && !nghttp2_session_want_write(session->ngh2)));
-}
+    apr_status_t status;
+    h2_stream *stream;
+    h2_ngheader *ngh;
+    int nid;
+    
+    ngh = h2_util_ngheader_make_req(is->pool, push->req);
+    nid = nghttp2_submit_push_promise(session->ngh2, 0, push->initiating_id, 
+                                      ngh->nv, ngh->nvlen, NULL);
+                                      
+    if (nid <= 0) {
+        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
+                      "h2_stream(%ld-%d): submitting push promise fail: %s",
+                      session->id, push->initiating_id, 
+                      nghttp2_strerror(nid));
+        return NULL;
+    }
+    
+    ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
+                  "h2_stream(%ld-%d): promised new stream %d for %s %s",
+                  session->id, push->initiating_id, nid,
+                  push->req->method, push->req->path);
+                  
+    stream = h2_session_open_stream(session, nid);
+    if (stream) {
+        h2_stream_set_h2_request(stream, is->id, push->req);
+        status = stream_schedule(session, stream, 1);
+        if (status != APR_SUCCESS) {
+            ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, session->c,
+                          "h2_stream(%ld-%d): scheduling push stream",
+                          session->id, stream->id);
+            h2_stream_cleanup(stream);
+            stream = NULL;
+        }
+    }
+    else {
+        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
+                      "h2_stream(%ld-%d): failed to create stream obj %d",
+                      session->id, push->initiating_id, nid);
+    }
 
-static int log_stream(void *ctx, h2_stream *stream)
-{
-    h2_session *session = (h2_session *)ctx;
-    AP_DEBUG_ASSERT(session);
-    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c,
-                  "h2_stream(%ld-%d): in set, suspended=%d, aborted=%d, "
-                  "has_data=%d",
-                  session->id, stream->id, stream->suspended, stream->aborted,
-                  h2_mplx_out_has_data_for(session->mplx, stream->id));
-    return 1;
+    if (!stream) {
+        /* try to tell the client that it should not wait. */
+        nghttp2_submit_rst_stream(session->ngh2, NGHTTP2_FLAG_NONE, nid,
+                                  NGHTTP2_INTERNAL_ERROR);
+    }
+    
+    return stream;
 }
 
-void h2_session_log_stats(h2_session *session)
+apr_status_t h2_session_stream_destroy(h2_session *session, h2_stream *stream)
 {
-    AP_DEBUG_ASSERT(session);
-    ap_log_cerror(APLOG_MARK, APLOG_TRACE1, 0, session->c,
-                  "h2_session(%ld): %d open streams",
-                  session->id, (int)h2_stream_set_size(session->streams));
-    h2_stream_set_iter(session->streams, log_stream, session);
+    apr_pool_t *pool = h2_stream_detach_pool(stream);
+
+    /* this may be called while the session has already freed
+     * some internal structures. */
+    if (session->mplx) {
+        h2_mplx_stream_done(session->mplx, stream->id, stream->rst_error);
+        if (session->last_stream == stream) {
+            session->last_stream = NULL;
+        }
+    }
+    
+    if (session->streams) {
+        h2_stream_set_remove(session->streams, stream->id);
+    }
+    h2_stream_destroy(stream);
+    
+    if (pool) {
+        apr_pool_clear(pool);
+        if (session->spare) {
+            apr_pool_destroy(session->spare);
+        }
+        session->spare = pool;
+    }
+    return APR_SUCCESS;
 }
 
 static int frame_print(const nghttp2_frame *frame, char *buffer, size_t maxlen)
@@ -1268,3 +1337,184 @@ static int frame_print(const nghttp2_fra
     }
 }
 
+int h2_session_push_enabled(h2_session *session)
+{
+    return nghttp2_session_get_remote_settings(session->ngh2, 
+                                               NGHTTP2_SETTINGS_ENABLE_PUSH);
+}
+
+
+apr_status_t h2_session_process(h2_session *session)
+{
+    apr_status_t status = APR_SUCCESS;
+    apr_interval_time_t wait_micros = 0;
+    static const int MAX_WAIT_MICROS = 200 * 1000;
+    int got_streams = 0;
+
+    while (!session->aborted && (nghttp2_session_want_read(session->ngh2)
+                                 || nghttp2_session_want_write(session->ngh2))) {
+        int have_written = 0;
+        int have_read = 0;
+                                 
+        /* Send data as long as we have it and window sizes allow. We are
+         * a server after all.
+         */
+        if (nghttp2_session_want_write(session->ngh2)) {
+            int rv;
+            
+            rv = nghttp2_session_send(session->ngh2);
+            if (rv != 0) {
+                ap_log_cerror( APLOG_MARK, APLOG_DEBUG, 0, session->c,
+                              "h2_session: send: %s", nghttp2_strerror(rv));
+                if (nghttp2_is_fatal(rv)) {
+                    h2_session_abort(session, status, rv);
+                    goto end_process;
+                }
+            }
+            else {
+                have_written = 1;
+                wait_micros = 0;
+            }
+        }
+        
+        if (wait_micros > 0) {
+            ap_log_cerror( APLOG_MARK, APLOG_TRACE3, 0, session->c,
+                          "h2_session: wait for data, %ld micros", (long)(wait_micros));
+            h2_conn_io_pass(&session->io);
+            status = h2_mplx_out_trywait(session->mplx, wait_micros, session->iowait);
+            
+            if (status == APR_TIMEUP) {
+                if (wait_micros < MAX_WAIT_MICROS) {
+                    wait_micros *= 2;
+                }
+            }
+        }
+        
+        if (nghttp2_session_want_read(session->ngh2))
+        {
+            /* When we
+             * - and have no streams at all
+             * - or have streams, but none is suspended or needs submit and
+             *   have nothing written on the last try
+             * 
+             * or, the other way around
+             * - have only streams where data can be sent, but could
+             *   not send anything
+             *
+             * then we are waiting on frames from the client (for
+             * example WINDOW_UPDATE or HEADER) and without new frames
+             * from the client, we cannot make any progress,
+             * 
+             * and *then* we can safely do a blocking read.
+             */
+            int may_block = (session->frames_received <= 1);
+            if (!may_block) {
+                if (got_streams) {
+                    may_block = (!have_written 
+                                 && !h2_stream_set_has_unsubmitted(session->streams)
+                                 && !h2_stream_set_has_suspended(session->streams));
+                }
+                else {
+                    may_block = 1;
+                }
+            }
+            
+            if (may_block) {
+                h2_conn_io_flush(&session->io);
+                if (session->c->cs) {
+                    session->c->cs->state = (got_streams? CONN_STATE_HANDLER
+                                             : CONN_STATE_WRITE_COMPLETION);
+                }
+                status = h2_conn_io_read(&session->io, APR_BLOCK_READ, 
+                                         session_receive, session);
+            }
+            else {
+                if (session->c->cs) {
+                    session->c->cs->state = CONN_STATE_HANDLER;
+                }
+                status = h2_conn_io_read(&session->io, APR_NONBLOCK_READ, 
+                                         session_receive, session);
+            }
+
+            switch (status) {
+                case APR_SUCCESS:       /* successful read, reset our idle timers */
+                    have_read = 1;
+                    wait_micros = 0;
+                    break;
+                case APR_EAGAIN:              /* non-blocking read, nothing there */
+                    break;
+                default:
+                    if (APR_STATUS_IS_ETIMEDOUT(status)
+                        || APR_STATUS_IS_ECONNABORTED(status)
+                        || APR_STATUS_IS_ECONNRESET(status)
+                        || APR_STATUS_IS_EOF(status)
+                        || APR_STATUS_IS_EBADF(status)) {
+                        /* common status for a client that has left */
+                        ap_log_cerror( APLOG_MARK, APLOG_DEBUG, status, session->c,
+                                      "h2_session(%ld): terminating",
+                                      session->id);
+                        /* Stolen from mod_reqtimeout to speed up lingering when
+                         * a read timeout happened.
+                         */
+                        apr_table_setn(session->c->notes, "short-lingering-close", "1");
+                    }
+                    else {
+                        /* uncommon status, log on INFO so that we see this */
+                        ap_log_cerror( APLOG_MARK, APLOG_INFO, status, session->c,
+                                      APLOGNO(02950) 
+                                      "h2_session(%ld): error reading, terminating",
+                                      session->id);
+                    }
+                    h2_session_abort(session, status, 0);
+                    goto end_process;
+            }
+        }
+        
+        got_streams = !h2_stream_set_is_empty(session->streams);
+        if (got_streams) {
+            h2_stream *stream;
+            
+            if (session->reprioritize) {
+                h2_mplx_reprioritize(session->mplx, stream_pri_cmp, session);
+                session->reprioritize = 0;
+            }
+            
+            if (!have_read && !have_written) {
+                /* Nothing read or written. That means no data yet ready to 
+                 * be send out. Slowly back off...
+                 */
+                if (wait_micros == 0) {
+                    wait_micros = 10;
+                }
+            }
+            
+            if (h2_stream_set_has_open_input(session->streams)) {
+                /* Check that any pending window updates are sent. */
+                status = h2_mplx_in_update_windows(session->mplx, update_window, session);
+                if (APR_STATUS_IS_EAGAIN(status)) {
+                    status = APR_SUCCESS;
+                }
+                else if (status == APR_SUCCESS) {
+                    /* need to flush window updates onto the connection asap */
+                    h2_conn_io_flush(&session->io);
+                }
+            }
+            
+            h2_session_resume_streams_with_data(session);
+            
+            if (h2_stream_set_has_unsubmitted(session->streams)) {
+                /* If we have responses ready, submit them now. */
+                while ((stream = h2_mplx_next_submit(session->mplx, session->streams))) {
+                    status = submit_response(session, stream);
+                }
+            }
+        }
+        
+        if (have_written) {
+            h2_conn_io_flush(&session->io);
+        }
+    }
+    
+end_process:
+    return status;
+}

Modified: httpd/httpd/branches/2.4.x/modules/http2/h2_session.h
URL: http://svn.apache.org/viewvc/httpd/httpd/branches/2.4.x/modules/http2/h2_session.h?rev=1715371&r1=1715370&r2=1715371&view=diff
==============================================================================
--- httpd/httpd/branches/2.4.x/modules/http2/h2_session.h (original)
+++ httpd/httpd/branches/2.4.x/modules/http2/h2_session.h Fri Nov 20 15:13:11 2015
@@ -41,6 +41,7 @@ struct apr_thread_mutext_t;
 struct apr_thread_cond_t;
 struct h2_config;
 struct h2_mplx;
+struct h2_push;
 struct h2_response;
 struct h2_session;
 struct h2_stream;
@@ -58,6 +59,8 @@ struct h2_session {
     request_rec *r;                 /* the request that started this in case
                                      * of 'h2c', NULL otherwise */
     int aborted;                    /* this session is being aborted */
+    int reprioritize;               /* scheduled streams priority needs to 
+                                     * be re-evaluated */
     apr_size_t frames_received;     /* number of http/2 frames received */
     apr_size_t max_stream_count;    /* max number of open streams */
     apr_size_t max_stream_mem;      /* max buffer memory for a single stream */
@@ -69,11 +72,14 @@ struct h2_session {
     h2_conn_io io;                  /* io on httpd conn filters */
     struct h2_mplx *mplx;           /* multiplexer for stream data */
     
+    struct h2_stream *last_stream;  /* last stream worked with */
     struct h2_stream_set *streams;  /* streams handled by this session */
     
     int max_stream_received;        /* highest stream id created */
     int max_stream_handled;         /* highest stream id handled successfully */
     
+    apr_pool_t *spare;              /* spare stream pool */
+    
     struct nghttp2_session *ngh2;   /* the nghttp2 session (internal use) */
     struct h2_workers *workers;     /* for executing stream tasks */
 };
@@ -102,6 +108,14 @@ h2_session *h2_session_rcreate(request_r
                                struct h2_workers *workers);
 
 /**
+ * Process the given HTTP/2 session until it is ended or a fatal
+ * error occured.
+ *
+ * @param session the sessionm to process
+ */
+apr_status_t h2_session_process(h2_session *session);
+
+/**
  * Destroy the session and all objects it still contains. This will not
  * destroy h2_task instances that have not finished yet. 
  * @param session the session to destroy
@@ -109,6 +123,13 @@ h2_session *h2_session_rcreate(request_r
 void h2_session_destroy(h2_session *session);
 
 /**
+ * Cleanup the session and all objects it still contains. This will not
+ * destroy h2_task instances that have not finished yet. 
+ * @param session the session to destroy
+ */
+void h2_session_eoc_callback(h2_session *session);
+
+/**
  * Called once at start of session. 
  * Sets up the session and sends the initial SETTINGS frame.
  * @param session the session to start
@@ -118,12 +139,6 @@ void h2_session_destroy(h2_session *sess
 apr_status_t h2_session_start(h2_session *session, int *rv);
 
 /**
- * Determine if session is finished.
- * @return != 0 iff session is finished and connection can be closed.
- */
-int h2_session_is_done(h2_session *session);
-
-/**
  * Called when an error occured and the session needs to shut down.
  * @param session the session to shut down
  * @param reason  the apache status that caused the shutdown
@@ -133,22 +148,15 @@ int h2_session_is_done(h2_session *sessi
 apr_status_t h2_session_abort(h2_session *session, apr_status_t reason, int rv);
 
 /**
- * Called before a session gets destroyed, might flush output etc. 
+ * Pass any buffered output data through the connection filters.
+ * @param session the session to flush
  */
-apr_status_t h2_session_close(h2_session *session);
+apr_status_t h2_session_flush(h2_session *session);
 
-/* Read more data from the client connection. Used normally with blocking
- * APR_NONBLOCK_READ, which will return APR_EAGAIN when no data is available.
- * Use with APR_BLOCK_READ only when certain that no data needs to be written
- * while waiting. */
-apr_status_t h2_session_read(h2_session *session, apr_read_type_e block);
-
-/* Write data out to the client, if there is any. Otherwise, wait for
- * a maximum of timeout micro-seconds and return to the caller. If timeout
- * occurred, APR_TIMEUP will be returned.
+/**
+ * Called before a session gets destroyed, might flush output etc. 
  */
-apr_status_t h2_session_write(h2_session *session,
-                              apr_interval_time_t timeout);
+apr_status_t h2_session_close(h2_session *session);
 
 /* Start submitting the response to a stream request. This is possible
  * once we have all the response headers. */
@@ -158,6 +166,39 @@ apr_status_t h2_session_handle_response(
 /* Get the h2_stream for the given stream idenrtifier. */
 struct h2_stream *h2_session_get_stream(h2_session *session, int stream_id);
 
-void h2_session_log_stats(h2_session *session);
+/**
+ * Create and register a new stream under the given id.
+ * 
+ * @param session the session to register in
+ * @param stream_id the new stream identifier
+ * @return the new stream
+ */
+struct h2_stream *h2_session_open_stream(h2_session *session, int stream_id);
+
+/**
+ * Returns if client settings have push enabled.
+ * @param != 0 iff push is enabled in client settings
+ */
+int h2_session_push_enabled(h2_session *session);
+
+/**
+ * Destroy the stream and release it everywhere. Reclaim all resources.
+ * @param session the session to which the stream belongs
+ * @param stream the stream to destroy
+ */
+apr_status_t h2_session_stream_destroy(h2_session *session, 
+                                       struct h2_stream *stream);
+
+/**
+ * Submit a push promise on the stream and schedule the new steam for
+ * processing..
+ * 
+ * @param session the session to work in
+ * @param is the stream initiating the push
+ * @param push the push to promise
+ * @return the new promised stream or NULL
+ */
+struct h2_stream *h2_session_push(h2_session *session, 
+                                  struct h2_stream *is, struct h2_push *push);
 
 #endif /* defined(__mod_h2__h2_session__) */