You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@httpd.apache.org by Marc Slemko <ma...@znep.com> on 1997/02/07 07:18:32 UTC

my version of the mod_include outbut buffering

Here is a cleaned up version of my way to add output buffering to
mod_include.  Speeds up some test files by a factor of 2
or 3 in testing here, although real world effects are far less.

Most of it is just changing some functions to take a request_rec
instead of a pool, since I need the request_rec for rwrite.  While 
these may be longer than Ed's patches, I think they are simpler
in concept and practice.  

I _think_ there are a few bugs in Ed's patches (that I have found...
could be some in mine to that I don't know about); the only one that 
I can say looks odd for sure is the bit dealing with the return from
rwrite, but I don't have the time to figure out if some of the other
things just look odd... if people prefer his patch, I will find time
to either find the bugs or convince myself I'm crazy.

Index: mod_include.c
===================================================================
RCS file: /export/home/cvs/apache/src/mod_include.c,v
retrieving revision 1.21
diff -c -r1.21 mod_include.c
*** mod_include.c	1997/01/20 04:28:13	1.21
--- mod_include.c	1997/02/07 05:52:25
***************
*** 75,82 ****
  #define SIZEFMT_KMG 1
  
  static void decodehtml(char *s);
! static char *get_tag(pool *p, FILE *in, char *tag, int tag_len, int dodecode);
! static int get_directive(FILE *in, char *d, pool *p);
  
  
  /* ------------------------ Environment function -------------------------- */
--- 75,82 ----
  #define SIZEFMT_KMG 1
  
  static void decodehtml(char *s);
! static char *get_tag(request_rec *r, FILE *in, char *tag, int tag_len, int dodecode);
! static int get_directive(FILE *in, char *d, request_rec *r);
  
  
  /* ------------------------ Environment function -------------------------- */
***************
*** 113,157 ****
      }
  }
  
! #define GET_CHAR(f,c,r,p) \
   { \
     int i = getc(f); \
!    if(feof(f) || ferror(f) || (i == -1)) { \
!         pfclose(p,f); \
!         return r; \
     } \
     c = (char)i; \
   }
  
  /* --------------------------- Parser functions --------------------------- */
  
! /* Grrrr... rputc makes this slow as all-get-out.  Elsewhere, it doesn't
!  * matter much, but this is an inner loop...
   */
  
  int find_string(FILE *in,char *str, request_rec *r, int printing) {
      int x,l=strlen(str),p;
      char c;
  
      p=0;
      while(1) {
!         GET_CHAR(in,c,1,r->pool);
          if(c == str[p]) {
!             if((++p) == l)
                  return 0;
          }
          else {
              if (printing) {
                  for(x=0;x<p;x++) {
!                     rputc(str[x],r);
                  }
!                 rputc(c,r);
              }
              p=0;
          }
      }
  }
  
  /*
   * decodes a string containing html entities or numeric character references.
   * 's' is overwritten with the decoded string.
--- 113,201 ----
      }
  }
  
! #define OUTBUFSIZE 4096
! /*
!  * f: file handle being read from
!  * c: character to read into
!  * ret: return value to use if input fails
!  * r: current request_rec
!  *
!  * Note that FLUSH_BUF() is redefined after find_string to do nothing
!  * since the other routines do not support it.  A lot of things in
!  * this module need to be rethought.
!  *
!  */
! #define GET_CHAR(f,c,ret,r) \
   { \
     int i = getc(f); \
!    if(i == EOF) { /* either EOF or error -- needs error handling if latter */ \
!        if (ferror(f)) \
! 	   fprintf(stderr, "encountered error in GET_CHAR macro, mod_include.\n"); \
!        FLUSH_BUF(r); \
!        pfclose(r->pool,f); \
!        return ret; \
     } \
     c = (char)i; \
   }
  
+ 
  /* --------------------------- Parser functions --------------------------- */
  
! /* PUT_CHAR and FLUSH_BUF currently only work within the scope of 
!  * find_string(); they are hacks to avoid calling rputc for each and
!  * every character output.  A common set of buffering calls for this 
!  * type of output SHOULD be implemented.
   */
+ #define PUT_CHAR(c,r) \
+  { \
+    outbuf[outind++] = c; \
+    if (outind == OUTBUFSIZE) { FLUSH_BUF(r) }; \
+  } 
+ 
+ /* there SHOULD be some error checking on the return value of
+  * rwrite, however it is unclear what the API for rwrite returning
+  * errors is and little can really be done to help the error in 
+  * any case.
+  */
+ #define FLUSH_BUF(r) \
+  { \
+    rwrite(outbuf, outind, r); \
+    outind = 0; \
+  }
  
  int find_string(FILE *in,char *str, request_rec *r, int printing) {
      int x,l=strlen(str),p;
+     char outbuf[OUTBUFSIZE];
+     int outind = 0;
      char c;
  
      p=0;
      while(1) {
!         GET_CHAR(in,c,1,r);
          if(c == str[p]) {
!             if((++p) == l) {
! 		FLUSH_BUF(r);
                  return 0;
+ 	    }
          }
          else {
              if (printing) {
                  for(x=0;x<p;x++) {
!                     PUT_CHAR(str[x],r);
                  }
!                 PUT_CHAR(c,r);
              }
              p=0;
          }
      }
  }
  
+ #undef FLUSH_BUF
+ #undef PUT_CHAR
+ /* FLUSH_BUF is called from within GET_CHAR, so still needs to exist 
+  * even if it does nothing */
+ #define FLUSH_BUF(r) {};
+ 
  /*
   * decodes a string containing html entities or numeric character references.
   * 's' is overwritten with the decoded string.
***************
*** 243,264 ****
   */
  
  static char *
! get_tag(pool *p, FILE *in, char *tag, int tagbuf_len, int dodecode) {
      char *t = tag, *tag_val, c, term;
      int n;
  
      n = 0;
  
      do { /* skip whitespace */
! 	GET_CHAR(in,c,NULL,p);
      } while (isspace(c));
  
      /* tags can't start with - */
      if(c == '-') {
!         GET_CHAR(in,c,NULL,p);
          if(c == '-') {
              do {
! 		GET_CHAR(in,c,NULL,p);
  	    } while (isspace(c));
              if(c == '>') {
                  strncpy(tag,"done", tagbuf_len-1);
--- 287,308 ----
   */
  
  static char *
! get_tag(request_rec *r, FILE *in, char *tag, int tagbuf_len, int dodecode) {
      char *t = tag, *tag_val, c, term;
      int n;
  
      n = 0;
  
      do { /* skip whitespace */
! 	GET_CHAR(in,c,NULL,r);
      } while (isspace(c));
  
      /* tags can't start with - */
      if(c == '-') {
!         GET_CHAR(in,c,NULL,r);
          if(c == '-') {
              do {
! 		GET_CHAR(in,c,NULL,r);
  	    } while (isspace(c));
              if(c == '>') {
                  strncpy(tag,"done", tagbuf_len-1);
***************
*** 277,296 ****
          }
  	if(c == '=' || isspace(c)) break;
  	*(t++) = tolower(c);
!         GET_CHAR(in,c,NULL,p);
      }
  
      *t++ = '\0';
      tag_val = t;
  
!     while (isspace(c)) GET_CHAR(in, c, NULL,p); /* space before = */
      if (c != '=') {
          ungetc(c, in);
          return NULL;
      }
  
      do {
! 	GET_CHAR(in,c,NULL,p);  /* space after = */
      } while (isspace(c));
  
      /* we should allow a 'name' as a value */
--- 321,340 ----
          }
  	if(c == '=' || isspace(c)) break;
  	*(t++) = tolower(c);
!         GET_CHAR(in,c,NULL,r);
      }
  
      *t++ = '\0';
      tag_val = t;
  
!     while (isspace(c)) GET_CHAR(in, c, NULL,r); /* space before = */
      if (c != '=') {
          ungetc(c, in);
          return NULL;
      }
  
      do {
! 	GET_CHAR(in,c,NULL,r);  /* space after = */
      } while (isspace(c));
  
      /* we should allow a 'name' as a value */
***************
*** 298,304 ****
      if (c != '"' && c != '\'') return NULL;
      term = c;
      while(1) {
! 	GET_CHAR(in,c,NULL,p);
  	if(++n == tagbuf_len) {
  	    t[tagbuf_len - 1] = '\0';
  	    return NULL;
--- 342,348 ----
      if (c != '"' && c != '\'') return NULL;
      term = c;
      while(1) {
! 	GET_CHAR(in,c,NULL,r);
  	if(++n == tagbuf_len) {
  	    t[tagbuf_len - 1] = '\0';
  	    return NULL;
***************
*** 306,312 ****
  /* Want to accept \" as a valid character within a string. */
  	if (c == '\\') {
  	    *(t++) = c; /* Add backslash */
! 	    GET_CHAR(in,c,NULL,p);
  	    if (c == term) /* Only if */
  		*(--t) = c; /* Replace backslash ONLY for terminator */
  	} else if (c == term) break;
--- 350,356 ----
  /* Want to accept \" as a valid character within a string. */
  	if (c == '\\') {
  	    *(t++) = c; /* Add backslash */
! 	    GET_CHAR(in,c,NULL,r);
  	    if (c == term) /* Only if */
  		*(--t) = c; /* Replace backslash ONLY for terminator */
  	} else if (c == term) break;
***************
*** 314,336 ****
      }
      *t = '\0';
      if (dodecode) decodehtml(tag_val);
!     return pstrdup (p, tag_val);
  }
  
  static int
! get_directive(FILE *in, char *d, pool *p) {
      char c;
  
      /* skip initial whitespace */
      while(1) {
!         GET_CHAR(in,c,1,p);
          if(!isspace(c))
              break;
      }
      /* now get directive */
      while(1) {
          *d++ = tolower(c);
!         GET_CHAR(in,c,1,p);
          if(isspace(c))
              break;
      }
--- 358,380 ----
      }
      *t = '\0';
      if (dodecode) decodehtml(tag_val);
!     return pstrdup (r->pool, tag_val);
  }
  
  static int
! get_directive(FILE *in, char *d, request_rec *r) {
      char c;
  
      /* skip initial whitespace */
      while(1) {
!         GET_CHAR(in,c,1,r);
          if(!isspace(c))
              break;
      }
      /* now get directive */
      while(1) {
          *d++ = tolower(c);
!         GET_CHAR(in,c,1,r);
          if(isspace(c))
              break;
      }
***************
*** 453,459 ****
      char *tag_val;
  
      while(1) {
!         if(!(tag_val = get_tag(r->pool, in, tag, MAX_STRING_LEN, 1)))
              return 1;
          if(!strcmp(tag,"file") || !strcmp (tag, "virtual")) {
  	    request_rec *rr=NULL;
--- 497,503 ----
      char *tag_val;
  
      while(1) {
!         if(!(tag_val = get_tag(r, in, tag, MAX_STRING_LEN, 1)))
              return 1;
          if(!strcmp(tag,"file") || !strcmp (tag, "virtual")) {
  	    request_rec *rr=NULL;
***************
*** 601,607 ****
      char parsed_string[MAX_STRING_LEN];
  
      while(1) {
!         if(!(tag_val = get_tag (r->pool, in, tag, MAX_STRING_LEN, 1)))
              return 1;
          if(!strcmp(tag,"cmd")) {
              parse_string(r, tag_val, parsed_string, MAX_STRING_LEN, 1);
--- 645,651 ----
      char parsed_string[MAX_STRING_LEN];
  
      while(1) {
!         if(!(tag_val = get_tag (r, in, tag, MAX_STRING_LEN, 1)))
              return 1;
          if(!strcmp(tag,"cmd")) {
              parse_string(r, tag_val, parsed_string, MAX_STRING_LEN, 1);
***************
*** 638,644 ****
      char *tag_val;
  
      while(1) {
!         if(!(tag_val = get_tag (r->pool, in, tag, MAX_STRING_LEN, 1)))
              return 1;
          if(!strcmp(tag,"var")) {
  	    char *val = table_get (r->subprocess_env, tag_val);
--- 682,688 ----
      char *tag_val;
  
      while(1) {
!         if(!(tag_val = get_tag (r, in, tag, MAX_STRING_LEN, 1)))
              return 1;
          if(!strcmp(tag,"var")) {
  	    char *val = table_get (r->subprocess_env, tag_val);
***************
*** 666,672 ****
      table *env = r->subprocess_env;
  
      while(1) {
!         if(!(tag_val = get_tag(r->pool, in, tag, MAX_STRING_LEN, 0)))
              return 1;
          if(!strcmp(tag,"errmsg")) {
              parse_string(r, tag_val, parsed_string, MAX_STRING_LEN, 0);
--- 710,716 ----
      table *env = r->subprocess_env;
  
      while(1) {
!         if(!(tag_val = get_tag(r, in, tag, MAX_STRING_LEN, 0)))
              return 1;
          if(!strcmp(tag,"errmsg")) {
              parse_string(r, tag_val, parsed_string, MAX_STRING_LEN, 0);
***************
*** 753,759 ****
      char parsed_string[MAX_STRING_LEN];
  
      while(1) {
!         if(!(tag_val = get_tag(r->pool, in, tag, MAX_STRING_LEN, 1)))
              return 1;
          else if(!strcmp(tag,"done"))
              return 0;
--- 797,803 ----
      char parsed_string[MAX_STRING_LEN];
  
      while(1) {
!         if(!(tag_val = get_tag(r, in, tag, MAX_STRING_LEN, 1)))
              return 1;
          else if(!strcmp(tag,"done"))
              return 0;
***************
*** 792,798 ****
      char parsed_string[MAX_STRING_LEN];
  
      while(1) {
!         if(!(tag_val = get_tag(r->pool, in, tag, MAX_STRING_LEN, 1)))
              return 1;
          else if(!strcmp(tag,"done"))
              return 0;
--- 836,842 ----
      char parsed_string[MAX_STRING_LEN];
  
      while(1) {
!         if(!(tag_val = get_tag(r, in, tag, MAX_STRING_LEN, 1)))
              return 1;
          else if(!strcmp(tag,"done"))
              return 0;
***************
*** 1386,1392 ****
      char *expr = '\0';
  
      while(1) {
!         tag_val = get_tag(r->pool, in, tag, MAX_STRING_LEN, 0);
          if(*tag == '\0')
              return 1;
          else if(!strcmp(tag,"done")) {
--- 1430,1436 ----
      char *expr = '\0';
  
      while(1) {
!         tag_val = get_tag(r, in, tag, MAX_STRING_LEN, 0);
          if(*tag == '\0')
              return 1;
          else if(!strcmp(tag,"done")) {
***************
*** 1416,1422 ****
      char *expr = '\0';
  
      while(1) {
!         tag_val = get_tag(r->pool, in, tag, MAX_STRING_LEN, 0);
          if(*tag == '\0')
              return 1;
          else if(!strcmp(tag,"done")) {
--- 1460,1466 ----
      char *expr = '\0';
  
      while(1) {
!         tag_val = get_tag(r, in, tag, MAX_STRING_LEN, 0);
          if(*tag == '\0')
              return 1;
          else if(!strcmp(tag,"done")) {
***************
*** 1451,1457 ****
      char tag[MAX_STRING_LEN];
      char *tag_val;
  
!     if(!(tag_val = get_tag(r->pool, in, tag, MAX_STRING_LEN, 1)))
          return 1;
      else if(!strcmp(tag,"done")) {
  #ifdef DEBUG_INCLUDE
--- 1495,1501 ----
      char tag[MAX_STRING_LEN];
      char *tag_val;
  
!     if(!(tag_val = get_tag(r, in, tag, MAX_STRING_LEN, 1)))
          return 1;
      else if(!strcmp(tag,"done")) {
  #ifdef DEBUG_INCLUDE
***************
*** 1473,1479 ****
      char tag[MAX_STRING_LEN];
      char *tag_val;
  
!     if(!(tag_val = get_tag(r->pool, in, tag, MAX_STRING_LEN, 1))) {
          return 1;
      } else if(!strcmp(tag,"done")) {
  #ifdef DEBUG_INCLUDE
--- 1517,1523 ----
      char tag[MAX_STRING_LEN];
      char *tag_val;
  
!     if(!(tag_val = get_tag(r, in, tag, MAX_STRING_LEN, 1))) {
          return 1;
      } else if(!strcmp(tag,"done")) {
  #ifdef DEBUG_INCLUDE
***************
*** 1497,1503 ****
  
      var = (char *)NULL;
      while (1) {
!         if(!(tag_val = get_tag(r->pool, in, tag, MAX_STRING_LEN, 1)))
              return 1;
          else if(!strcmp(tag,"done"))
              return 0;
--- 1541,1547 ----
  
      var = (char *)NULL;
      while (1) {
!         if(!(tag_val = get_tag(r, in, tag, MAX_STRING_LEN, 1)))
              return 1;
          else if(!strcmp(tag,"done"))
              return 0;
***************
*** 1523,1529 ****
      table_entry *elts = (table_entry *) r->subprocess_env->elts;
      int i;
  
!     if(!(tag_val = get_tag(r->pool, in, tag, MAX_STRING_LEN, 1)))
          return 1;
      else if(!strcmp(tag,"done")) {
              for (i = 0; i < r->subprocess_env->nelts; ++i)
--- 1567,1573 ----
      table_entry *elts = (table_entry *) r->subprocess_env->elts;
      int i;
  
!     if(!(tag_val = get_tag(r, in, tag, MAX_STRING_LEN, 1)))
          return 1;
      else if(!strcmp(tag,"done")) {
              for (i = 0; i < r->subprocess_env->nelts; ++i)
***************
*** 1572,1578 ****
  
      while(1) {
          if(!find_string(f,STARTING_SEQUENCE,r,printing)) {
!             if(get_directive(f,directive,r->pool))
                  return;
              if(!strcmp(directive,"if")) {
                  if (!printing) {
--- 1616,1622 ----
  
      while(1) {
          if(!find_string(f,STARTING_SEQUENCE,r,printing)) {
!             if(get_directive(f,directive,r))
                  return;
              if(!strcmp(directive,"if")) {
                  if (!printing) {


Re: my version of the mod_include outbut buffering

Posted by Marc Slemko <ma...@znep.com>.
On Fri, 7 Feb 1997, Ed Korthof wrote:

> I'm not particularly invested in my patch for mod_include (and I just realized,
> so I'll remove it from the running.  However, I would suggest a few changes to
> Marc's patch, specifically to avoid having the change the other functions which
> use GET_CHAR.  IMO, changing the arguments they take is something we should
> avoid.
> 
> Marc, why don't we use the GET_CHAR, PUT_CHAR, and FLUSH_BUFFER macros you
> posted -- define them above find_string -- and undef them below find_string;
> then define the old GET_CHAR macro for use by the other functions?  Not the
> cleanest code, but it should do the job efficiently.

That's ugly.  Oh yes, so is my patch...  Hey, so is the whole module. 
Heck, the entire _concept_ is ugly.  <sigh>

I agree.  I thought about doing it before but didn't.  My new patch.  I am
just guessing at a size of 4k for the output buffer; seems reasonable to
me...

In general, functions that pass fields of a request_rec instead of just
passing the request_rec annoy me. Yes, yes, it is cheaper to not pass the
whole thing but it sure saves changes later... 

New patch:

Index: mod_include.c
===================================================================
RCS file: /export/home/cvs/apache/src/mod_include.c,v
retrieving revision 1.21
diff -c -r1.21 mod_include.c
*** mod_include.c	1997/01/20 04:28:13	1.21
--- mod_include.c	1997/02/08 02:49:56
***************
*** 113,156 ****
      }
  }
  
! #define GET_CHAR(f,c,r,p) \
   { \
     int i = getc(f); \
!    if(feof(f) || ferror(f) || (i == -1)) { \
!         pfclose(p,f); \
!         return r; \
     } \
     c = (char)i; \
   }
  
- /* --------------------------- Parser functions --------------------------- */
- 
- /* Grrrr... rputc makes this slow as all-get-out.  Elsewhere, it doesn't
-  * matter much, but this is an inner loop...
-  */
- 
  int find_string(FILE *in,char *str, request_rec *r, int printing) {
      int x,l=strlen(str),p;
      char c;
  
      p=0;
      while(1) {
!         GET_CHAR(in,c,1,r->pool);
          if(c == str[p]) {
!             if((++p) == l)
                  return 0;
          }
          else {
              if (printing) {
                  for(x=0;x<p;x++) {
!                     rputc(str[x],r);
                  }
!                 rputc(c,r);
              }
              p=0;
          }
      }
  }
  
  /*
   * decodes a string containing html entities or numeric character references.
--- 113,209 ----
      }
  }
  
! 
! 
! /* --------------------------- Parser functions --------------------------- */
! 
! #define OUTBUFSIZE 4096
! /* PUT_CHAR and FLUSH_BUF currently only work within the scope of 
!  * find_string(); they are hacks to avoid calling rputc for each and
!  * every character output.  A common set of buffering calls for this 
!  * type of output SHOULD be implemented.
!  */
! #define PUT_CHAR(c,r) \
!  { \
!    outbuf[outind++] = c; \
!    if (outind == OUTBUFSIZE) { FLUSH_BUF(r) }; \
!  } 
! 
! /* there SHOULD be some error checking on the return value of
!  * rwrite, however it is unclear what the API for rwrite returning
!  * errors is and little can really be done to help the error in 
!  * any case.
!  */
! #define FLUSH_BUF(r) \
!  { \
!    rwrite(outbuf, outind, r); \
!    outind = 0; \
!  }
! 
! /*
!  * f: file handle being read from
!  * c: character to read into
!  * ret: return value to use if input fails
!  * r: current request_rec
!  *
!  * This macro is redefined after find_string() for historical reasons
!  * to avoid too many code changes.  This is one of the many things
!  * that should be fixed.
!  */
! #define GET_CHAR(f,c,ret,r) \
   { \
     int i = getc(f); \
!    if(i == EOF) { /* either EOF or error -- needs error handling if latter */ \
!        if (ferror(f)) \
! 	   fprintf(stderr, "encountered error in GET_CHAR macro, mod_include.\n"); \
!        FLUSH_BUF(r); \
!        pfclose(r->pool,f); \
!        return ret; \
     } \
     c = (char)i; \
   }
  
  int find_string(FILE *in,char *str, request_rec *r, int printing) {
      int x,l=strlen(str),p;
+     char outbuf[OUTBUFSIZE];
+     int outind = 0;
      char c;
  
      p=0;
      while(1) {
!         GET_CHAR(in,c,1,r);
          if(c == str[p]) {
!             if((++p) == l) {
! 		FLUSH_BUF(r);
                  return 0;
+ 	    }
          }
          else {
              if (printing) {
                  for(x=0;x<p;x++) {
!                     PUT_CHAR(str[x],r);
                  }
!                 PUT_CHAR(c,r);
              }
              p=0;
          }
      }
  }
+ 
+ #undef FLUSH_BUF
+ #undef PUT_CHAR
+ #undef GET_CHAR
+ #define GET_CHAR(f,c,r,p) \
+  { \
+    int i = getc(f); \
+    if(i == EOF) { /* either EOF or error -- needs error handling if latter */ \
+        if (ferror(f)) \
+ 	   fprintf(stderr, "encountered error in GET_CHAR macro, mod_include.\n"); \
+        pfclose(p,f); \
+        return r; \
+    } \
+    c = (char)i; \
+  }
  
  /*
   * decodes a string containing html entities or numeric character references.


Re: my version of the mod_include outbut buffering

Posted by Ed Korthof <ed...@organic.com>.
I'm not particularly invested in my patch for mod_include (and I just realized,
so I'll remove it from the running.  However, I would suggest a few changes to
Marc's patch, specifically to avoid having the change the other functions which
use GET_CHAR.  IMO, changing the arguments they take is something we should
avoid.

Marc, why don't we use the GET_CHAR, PUT_CHAR, and FLUSH_BUFFER macros you
posted -- define them above find_string -- and undef them below find_string;
then define the old GET_CHAR macro for use by the other functions?  Not the
cleanest code, but it should do the job efficiently.

-- 
     -- Ed Korthof        |  Web Server Engineer --
     -- ed@organic.com    |  Organic Online, Inc --
     -- (415) 278-5676    |  Fax: (415) 284-6891 --