You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by jp...@apache.org on 2013/03/15 04:22:25 UTC
[1/2] TS-1748: Add jtest to the build
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8ad2978/tools/jtest/jtest.cc
----------------------------------------------------------------------
diff --git a/tools/jtest/jtest.cc b/tools/jtest/jtest.cc
index da7236a..d7e5408 100644
--- a/tools/jtest/jtest.cc
+++ b/tools/jtest/jtest.cc
@@ -54,9 +54,6 @@
typedef int64_t ink_hrtime;
-#define bool int
-#define false 0
-#define true 1
#define SIZE(x) (sizeof(x)/sizeof((x)[0]))
/*
@@ -115,6 +112,10 @@ extern "C" int gethostname(char *name, int namelen);
#define MAX_DEFERED_URLS 10000
#define DEFERED_URLS_BLOCK 2000
+static const char * hexdigits = "0123456789ABCDEFabcdef";
+static const char * dontunescapify = "#;/?+=&:@%";
+static const char * dontescapify = "#;/?+=&:@~.-_%";
+
enum FTP_MODE {
FTP_NULL,
FTP_PORT,
@@ -126,13 +127,16 @@ typedef int (*poll_cb)(int);
struct ArgumentDescription;
typedef void ArgumentFunction(
ArgumentDescription * argument_descriptions, int n_argument_descriptions,
- char * arg);
+ const char * arg);
+
+static void jtest_usage(ArgumentDescription * argument_descriptions,
+ int n_argument_descriptions, const char * arg);
static int read_request(int sock);
static int write_request(int sock);
static int make_client (unsigned int addr, int port);
static void make_bfc_client (unsigned int addr, int port);
-static int make_url_client(char * url,char * base_url = 0, bool seen = false,
+static int make_url_client(const char * url,const char * base_url = 0, bool seen = false,
bool unthrottled = false);
static int write_ftp_response(int sock);
static void interval_report();
@@ -231,7 +235,6 @@ uint64_t total_server_response_header_bytes = 0;
uint64_t total_proxy_response_body_bytes = 0;
uint64_t total_proxy_response_header_bytes = 0;
ink_hrtime now = 0, start_time = 0;
-ArgumentFunction jtest_usage;
int extra_headers = 0;
int alternates = 0;
int abort_retry_speed = 0;
@@ -248,12 +251,12 @@ double zipf = 0.0;
int zipf_bucket_size = 1;
struct ArgumentDescription {
- char * name;
+ const char * name;
char key;
- char * description;
- char * type;
- void * location;
- char * env;
+ const char * description;
+ const char * type;
+ const void * location;
+ const char * env;
ArgumentFunction * pfn;
};
@@ -532,4182 +535,7 @@ class ParseRules
ParseRules & operator = (const ParseRules &);
};
-extern const unsigned int parseRulesCType[];
-extern const char parseRulesCTypeToUpper[];
-extern const char parseRulesCTypeToLower[];
-
-inline int ParseRules::is_char(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 0)) ? (1) : (0));
-}
-inline int ParseRules::is_upalpha(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 1)) ? (1) : (0));
-}
-inline int ParseRules::is_loalpha(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 2)) ? (1) : (0));
-}
-inline int ParseRules::is_alpha(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 3)) ? (1) : (0));
-}
-inline int ParseRules::is_digit(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 4)) ? (1) : (0));
-}
-inline int ParseRules::is_alnum(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 26)) ? (1) : (0));
-}
-inline int ParseRules::is_ctl(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 5)) ? (1) : (0));
-}
-inline int ParseRules::is_ws(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 6)) ? (1) : (0));
-}
-inline int ParseRules::is_hex(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 7)) ? (1) : (0));
-}
-inline int ParseRules::is_cr(char c) {
- return (c == CHAR_CR);
-}
-inline int ParseRules::is_lf(char c) {
- return (c == CHAR_LF);
-}
-inline int ParseRules::is_splf(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 19)) ? (1) : (0));
-}
-inline int ParseRules::is_spcr(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 18)) ? (1) : (0));
-}
-inline int ParseRules::is_wslfcr(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 20)) ? (1) : (0));
-}
-inline int ParseRules::is_extra(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 9)) ? (1) : (0));
-}
-inline int ParseRules::is_safe(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 10)) ? (1) : (0));
-}
-inline int ParseRules::is_unsafe(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 11)) ? (1) : (0));
-}
-inline int ParseRules::is_reserved(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 13)) ? (1) : (0));
-}
-inline int ParseRules::is_national(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 12)) ? (1) : (0));
-}
-inline int ParseRules::is_unreserved(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 14)) ? (1) : (0));
-}
-inline int ParseRules::is_punct(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 15)) ? (1) : (0));
-}
-inline int ParseRules::is_end_of_url(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 16)) ? (1) : (0));
-}
-inline int ParseRules::is_escape(const char *seq) {
- return (seq[0] == '%' && is_hex(seq[1]) && is_hex(seq[2]));
-}
-inline int ParseRules::is_uchar(const char *seq) {
- return (is_unreserved(seq[0]) || is_escape(seq));
-}
-inline int ParseRules::is_pchar(const char *seq) {
- if (*seq != '%')
- return ((parseRulesCType[*seq] & (1 << 8)) ? (1) : (0));
- else
- return is_hex(seq[1]) && is_hex(seq[2]);
-}
-inline int ParseRules::is_tspecials(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 17)) ? (1) : (0));
-}
-inline int ParseRules::is_token(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 22)) ? (1) : (0));
-}
-inline char ParseRules::ink_toupper(char c) {
- return parseRulesCTypeToUpper[(unsigned char) c];
-}
-inline char ParseRules::ink_tolower(char c) {
- return parseRulesCTypeToLower[(unsigned char) c];
-}
-inline int ParseRules::is_eow(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 21)) ? (1) : (0));
-}
-inline int ParseRules::is_wildmat(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 23)) ? (1) : (0));
-}
-inline int ParseRules::is_sep(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 24)) ? (1) : (0));
-}
-inline int ParseRules::is_empty(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 25)) ? (1) : (0));
-}
-inline int ParseRules::is_space(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 27)) ? (1) : (0));
-}
-inline int ParseRules::is_control(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 28)) ? (1) : (0));
-}
-inline int ParseRules::is_mime_sep(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 29)) ? (1) : (0));
-}
-inline int ParseRules::is_http_field_name(char c) {
- return ((parseRulesCType[(unsigned char) c] & (1 << 30)) ? (1) : (0));
-}
-inline int ParseRules::is_http_field_value(char c) {
- return ((parseRulesCType[(unsigned char) c] & (((unsigned int)1) << 31)) ? (1) : (0));
-}
-inline int ParseRules::strncasecmp_eow(
- const char *s1, const char *s2, int count)
-{
- for (int i = 0; i < count; i++) {
- const char &a = s1[i];
- const char &b = s2[i];
- if (ink_tolower(a) != ink_tolower(b))
- return (is_eow(a) && is_eow(b));
- }
- return (1);
-}
-inline int ParseRules::strlen_eow(const char *s){
- for (int i = 0; 1; i++) {
- if (is_eow(s[i]))
- return (i);
- }
-}
-inline const char *ParseRules::strstr_eow(const char *s1, const char *s2) {
- int i1;
- int s2_len = strlen_eow(s2);
- for (i1 = 0; !is_eow(s1[i1]); i1++)
- if (ink_tolower(s1[i1]) == ink_tolower(s2[0]))
- if (strncasecmp_eow(&s1[i1], &s2[0], s2_len))
- return (&s1[i1]);
- return (0);
-}
-inline const char *ParseRules::
-strcasestr(const char *s1, const char *s2) {
- int i1;
- int s2_len = strlen(s2);
- for (i1 = 0; s1[i1] != '\0'; i1++)
- if (ink_tolower(s1[i1]) == ink_tolower(s2[0]))
- if (strncasecmp_eow(&s1[i1], &s2[0], s2_len))
- return (&s1[i1]);
- return (0);
-}
-inline const char *ParseRules::memchr(const char *s, char c, int max_length) {
- for (int i = 0; i < max_length; i++)
- if (s[i] == c)
- return (&s[i]);
- return (0);
-}
-inline const char *ParseRules::strchr(const char *s, char c) {
- for (int i = 0; s[i] != '\0'; i++)
- if (s[i] == c)
- return (&s[i]);
- return (0);
-}
-inline int
-ink_atoi(const char *str) {
- int num = 0;
- int negative = 0;
- while (*str && ParseRules::is_wslfcr(*str))
- str += 1;
- if (*str == '-') {
- negative = 1;
- str += 1;
- }
- while (*str && ParseRules::is_digit(*str))
- num = (num * 10) - (*str++ - '0');
- if (!negative)
- num = -num;
- return num;
-}
-inline unsigned int
-ink_atoui(const char *str) {
- unsigned int num = 0;
- while (*str && ParseRules::is_wslfcr(*str))
- str += 1;
- while (*str && ParseRules::is_digit(*str))
- num = (num * 10) + (*str++ - '0');
- return num;
-}
-inline int64_t
-ink_atoll(const char *str) {
- int64_t num = 0;
- int negative = 0;
- while (*str && ParseRules::is_wslfcr(*str))
- str += 1;
- if (*str == '-') {
- negative = 1;
- str += 1;
- }
- while (*str && ParseRules::is_digit(*str))
- num = (num * 10) - (*str++ - '0');
- if (!negative)
- num = -num;
- return num;
-}
-
-#define HRTIME_FOREVER (10*HRTIME_DECADE)
-#define HRTIME_DECADE (10*HRTIME_YEAR)
-#define HRTIME_YEAR (365*HRTIME_DAY+HRTIME_DAY/4)
-#define HRTIME_WEEK (7*HRTIME_DAY)
-#define HRTIME_DAY (24*HRTIME_HOUR)
-#define HRTIME_HOUR (60*HRTIME_MINUTE)
-#define HRTIME_MINUTE (60*HRTIME_SECOND)
-#define HRTIME_SECOND (1000*HRTIME_MSECOND)
-#define HRTIME_MSECOND (1000*HRTIME_USECOND)
-#define HRTIME_USECOND (1000*HRTIME_NSECOND)
-#define HRTIME_NSECOND (1LL)
-#define MAX_FILE_ARGUMENTS 100
-
-char * file_arguments[MAX_FILE_ARGUMENTS] = { 0 };
-char * program_name = (char *)"inktomi";
-int n_file_arguments = 0;
-
-static char * argument_types_keys = (char *)"ISDfFTL";
-static char * argument_types_descriptions[] = {
- (char *)"int ",
- (char *)"str ",
- (char *)"dbl ",
- (char *)"off ",
- (char *)"on ",
- (char *)"tog ",
- (char *)"i64 ",
- (char *)" "
-};
-
-void show_argument_configuration(ArgumentDescription * argument_descriptions,
- int n_argument_descriptions);
-
-void usage(ArgumentDescription * argument_descriptions,
- int n_argument_descriptions,
- char * arg_unused);
-
-void process_args(ArgumentDescription * argument_descriptions,
- int n_argument_descriptions,
- char **argv);
-
-#ifdef RELEASE
-#define ink_assert(EX) (void)(EX)
-#define ink_debug_assert(EX)
-#else
-#define ink_assert(EX) (\
- void)((EX) || (_ink_assert(#EX, __FILE__, __LINE__)))
-#define ink_debug_assert(EX) \
- (void)((EX) || (_ink_assert(#EX, __FILE__, __LINE__)))
-#endif
-
-#define ink_release_assert(EX) \
- (void)((EX) || (_ink_assert(#EX, __FILE__, __LINE__)))
-
-int _ink_assert(const char * a, const char * f, int l);
-void ink_fatal(int return_code, const char *message_format, ...);
-void ink_warning(const char *message_format, ...);
-#define min(_a,_b) ((_a)<(_b)?(_a):(_b))
-
-inline ink_hrtime ink_get_hrtime()
-{
-# if defined(FreeBSD)
- timespec ts;
- clock_gettime(CLOCK_REALTIME, &ts);
- return (ts.tv_sec * HRTIME_SECOND + ts.tv_nsec * HRTIME_NSECOND);
-# else
- timeval tv;
- gettimeofday(&tv,NULL);
- return (tv.tv_sec * HRTIME_SECOND + tv.tv_usec * HRTIME_USECOND);
-# endif
-}
-
-typedef struct {
- char sche[MAX_URL_LEN + 1];
- char host[MAX_URL_LEN + 1];
- char port[MAX_URL_LEN + 1];
- char path[MAX_URL_LEN + 1];
- char frag[MAX_URL_LEN + 1];
- char quer[MAX_URL_LEN + 1];
- char para[MAX_URL_LEN + 1];
-
- int sche_exists;
- int host_exists;
- int port_exists;
- int path_exists;
- int frag_exists;
- int quer_exists;
- int para_exists;
-
- int rel_url;
- int leading_slash;
- int is_path_name;
-} InkWebURLComponents;
-
-static int ink_web_remove_dots(
- char *src, char *dest, int *leadingslash, int max_dest_len);
-
-static int ink_web_unescapify_string(
- char *dest_in, char *src_in, int max_dest_len);
-
-int ink_web_escapify_string(char *dest_in, char *src_in, int max_dest_len);
-
-static void ink_web_decompose_url(
- char *src_url,
- char *sche, char *host, char *port, char *path,
- char *frag, char *quer, char *para,
- int *real_sche_exists, int *real_host_exists,
- int *real_port_exists, int *real_path_exists,
- int *real_frag_exists, int *real_quer_exists,
- int *real_para_exists,
- int *real_relative_url, int *real_leading_slash);
-
-static void ink_web_canonicalize_url(char *base_url, char *emb_url,
- char *dest_url, int max_dest_url_len);
-
-static void ink_web_decompose_url_into_structure(char *url,
- InkWebURLComponents *c);
-
-static void remove_last_seg(char *src, char *dest)
-{
- char *ptr;
- for (ptr = src + strlen(src) - 1; ptr >= src; ptr--)
- if (*ptr == '/') break;
- while (src <= ptr) *dest++ = *src++;
- *dest = '\0';
-}
-
-static inline void remove_multiple_slash(char *src, char *dest)
-{
- char *ptr=NULL;
-
- for (ptr = src; *ptr;) {
- *(dest++) = *ptr;
- if (*ptr == '/') {
- while ((*ptr== '/') && *ptr) {
- ptr++;
- }
- } else {
- ptr++;
- }
- }
- *dest = '\0';
-}
-
-static inline void append_string(char *dest, char *src, int *offset_ptr,
- int max_len)
-{
- int num = strlen(src);
- if (*offset_ptr + num > max_len)
- num = max_len-*offset_ptr;
- strncpy(dest+*offset_ptr, src, num+1);
- (*offset_ptr) += num;
-}
-
-// End Library functions
-
-void show_version() {
- char b[] = VERSION_NUM;
- char * v = strchr(b,':');
- v += 2;
- *strchr(v,'$') = 0;
- printf(VERSION, v);
-}
-
-void jtest_usage(ArgumentDescription * argument_descriptions,
- int n_argument_descriptions, char * arg)
-{
- show_version();
- usage(argument_descriptions, n_argument_descriptions, arg);
-}
-
-static void panic(char * s) {
- fputs(s, stderr);
- exit(1);
-}
-
-static void panic_perror(char * s) {
- perror(s);
- exit(1);
-}
-
-int max_limit_fd() {
- struct rlimit rl;
- if (getrlimit(RLIMIT_NOFILE,&rl) >= 0) {
-#ifdef OPEN_MAX
- // Darwin
- rl.rlim_cur = min(OPEN_MAX, rl.rlim_max);
-#else
- rl.rlim_cur = rl.rlim_max;
-#endif
- if (setrlimit(RLIMIT_NOFILE,&rl) >= 0) {
- if (getrlimit(RLIMIT_NOFILE,&rl) >= 0) {
- return rl.rlim_cur;
- }
- }
- }
- panic_perror("couldn't set RLIMIT_NOFILE\n");
- return -1;
-}
-
-int read_ready(int fd) {
- struct pollfd p;
- p.events = POLLIN;
- p.fd = fd;
- int r = poll(&p,1,0);
- if (r<=0) return r;
- if (p.revents & (POLLERR|POLLNVAL)) return -1;
- if (p.revents & (POLLIN|POLLHUP)) return 1;
- return 0;
-}
-
-static void poll_init(int sock) {
- if (!fd[sock].req_header)
- fd[sock].req_header = (char*)malloc(HEADER_SIZE * pipeline);
- if (!fd[sock].response_header)
- fd[sock].response_header = (char*)malloc(HEADER_SIZE);
- if (!fd[sock].base_url)
- fd[sock].base_url = (char*)malloc(HEADER_SIZE);
- fd[sock].reset();
-}
-
-static void poll_set(int sock, poll_cb read_cb, poll_cb write_cb = NULL) {
- if (verbose) printf("adding poll %d\n", sock);
- fd[sock].fd = sock;
- fd[sock].read_cb = read_cb;
- fd[sock].write_cb = write_cb;
- if (last_fd < sock)
- last_fd = sock;
-}
-
-static void poll_init_set(int sock, poll_cb read_cb, poll_cb write_cb = NULL) {
- poll_init(sock);
- poll_set(sock, read_cb, write_cb);
-}
-
-static int fast(int sock, int speed, int d) {
- if (!speed) return 0;
- int64_t t = now - fd[sock].start + 1;
- int target = (int)(((t / HRTIME_MSECOND) * speed) / 1000);
- int delta = d - target;
- if (delta > 0) {
- int mwait = (delta * 1000) / speed;
- fd[sock].ready = now + (mwait * HRTIME_MSECOND);
- return 1;
- } else
- fd[sock].ready = now;
- return 0;
-}
-
-static int faster_than(int sock, int speed, int d) {
- if (!speed) return 1;
- int64_t t = now - fd[sock].start + 1;
- int target = (int)(((t / HRTIME_MSECOND) * speed) / 1000);
- int delta = d - target;
- if (delta > 0)
- return 1;
- return 0;
-}
-
-static void get_path_from_req(char * buf,char ** purl_start, char ** purl_end)
-{
- char * url_start = buf;
- char * url_end = NULL;
- if (!strncasecmp(url_start, "GET ", sizeof("GET ")-1)) {
- url_start += sizeof("GET ")-1;
- url_end = (char*)memchr(url_start,' ',70);
- } else
- url_end = (char*)memchr(url_start,0,70);
- if (!url_end) panic("malformed request\n");
- if (url_end - url_start > 10)
- if (!strncasecmp(url_start, "http://", sizeof("http://")-1)) {
- url_start += sizeof("http://")-1;
- url_start = (char*)memchr(url_start,'/',70);
- }
- *purl_start = url_start;
- *purl_end = url_end;
-}
-
-static int send_response (int sock) {
- int err, towrite;
-
- if (fd[sock].req_pos >= 0) {
- char header[1024], *url_end = NULL, *url_start = NULL;
- int url_len = 0;
- char* content_type;
- switch (server_content_type) {
- case 1: content_type = "text/html"; break;
- case 2: content_type = "image/jpeg"; break;
- default:
- content_type =
- (char*)((compd_suite || alternates) ? "image/jpeg" : "text/html");
- if (only_server && strstr(fd[sock].req_header, "Cookie:"))
- content_type = "image/jpeg";
- }
- if (!ftp && embed_url && fd[sock].response_length > 16) {
- get_path_from_req(fd[sock].req_header, &url_start, &url_end);
- *url_end = 0;
- url_len = url_end - url_start;
- }
- int print_len = 0;
- if (!ftp) {
- if (fd[sock].ims) {
- print_len = sprintf(
- header,"HTTP/1.0 304 Not-Modified\r\n"
- "Content-Type: %s\r\n"
- "Last-Modified: Mon, 05 Oct 2010 01:00:00 GMT\r\n"
- "%s"
- "\r\n",
- content_type,
- fd[sock].keepalive>0?"Connection: Keep-Alive\r\n":"");
- url_len = 0;
- } else
- print_len = sprintf(
- header,"HTTP/1.0 200 OK\r\n"
- "Content-Type: %s\r\n"
- "Cache-Control: max-age=630720000\r\n"
- "Last-Modified: Mon, 05 Oct 2010 01:00:00 GMT\r\n"
- "%s"
- "Content-Length: %d\r\n"
- "%s"
- "\r\n%s",
- content_type,
- fd[sock].keepalive>0?"Connection: Keep-Alive\r\n":"",
- fd[sock].response_length,
- no_cache?"Pragma: no-cache\r\nCache-Control: no-cache\r\n":"",
- url_start ? url_start : "");
- } else
- url_len = print_len =
- sprintf(header, "ftp://%s:%d/%12.10f/%d",
- local_host, server_port,
- fd[sock].doc, fd[sock].length);
- if (show_headers) printf("Response to Proxy: {\n%s}\n", header);
- int len = print_len - fd[sock].req_pos;
- ink_assert(len>0);
- do {
- err = write(sock, header + fd[sock].req_pos, len);
- } while ((err == -1) && (errno == EINTR));
- if (err <= 0) {
- if (!err) return -1;
- if (errno == EAGAIN || errno == ENOTCONN) return 0;
- return -1;
- }
- if (verbose) printf("wrote %d %d\n", sock, err);
- new_tbytes += err;
- fd[sock].req_pos += err;
- fd[sock].bytes += err;
- if (fd[sock].req_pos >= len)
- fd[sock].req_pos = -1;
- else
- return 0;
- fd[sock].response += url_len;
- fd[sock].length -= url_len;
- total_server_response_header_bytes += print_len - url_len;
- total_server_response_body_bytes += url_len;
- }
-
- /* then the response */
- towrite = server_speed?server_speed:MAX_RESPONSE_LENGTH;
- if (fd[sock].length < towrite)
- towrite = fd[sock].length;
- if (towrite > 0) {
- if (fast(sock,server_speed,fd[sock].bytes))
- return 0;
- do {
- err = write (sock, fd[sock].response, towrite);
- } while ((err == -1) && (errno == EINTR));
- if (err < 0) {
- if (errno == EAGAIN || errno == ENOTCONN) return 0;
- fprintf(stderr,"write errno %d length %d sock %d\n",errno,towrite,sock);
- errors++;
- return -1;
- }
- if (verbose) printf("wrote %d %d\n", sock, err);
-
- new_tbytes += err;
- total_server_response_body_bytes += err;
- fd[sock].response += err;
- fd[sock].length -= err;
- fd[sock].bytes += err;
- }
-
- if (fast(sock,server_speed,fd[sock].bytes))
- return 0;
- if (fd[sock].length <= 0 || !err) {
- if (fd[sock].response)
- new_sops++;
- if (verbose) printf("write %d done\n", sock);
- if (fd[sock].keepalive > 0 && !ftp) {
- poll_init_set(sock, read_request);
- fd[sock].start = now;
- fd[sock].ready = now + server_delay * HRTIME_MSECOND;
- return 0;
- }
- return 1;
- }
-
- return 0;
-}
-
-char * strncasestr (char *s, char * find, int len) {
- int findlen = strlen(find);
- char * e = s + len;
- while (1) {
- char * x = (char*)memchr(s,*find,e - s);
- if (!x) {
- if (ParseRules::is_upalpha(*find))
- x = (char*)memchr(s,ParseRules::ink_tolower(*find),e - s);
- else
- x = (char*)memchr(s,ParseRules::ink_toupper(*find),e - s);
- if (!x) break;
- }
- if (!strncasecmp(find, x, findlen)) return x;
- s = x + 1;
- }
- return NULL;
-}
-
-static char * check_keepalive(char * r, int length) {
- char * ka = strncasestr(r,"Connection:", length);
- if (ka) {
- int l = length - (ka - r);
- char * e = (char*)memchr(ka,'\n',l);
- if (!e) e = (char*)memchr(ka,'\r',l);
- if (strncasestr(ka,"close",e-ka))
- return NULL;
- }
- return ka;
-}
-
-static int check_alt(char * r, int length) {
- char * s = strncasestr(r, "Cookie:", length);
- if (!s) {
- s = strncasestr(r, "User-Agent:", length);
- if (s) s += sizeof("User-Agent:");
- } else
- s += sizeof("Cookie:");
- if (s) {
- int l = length - (s - r);
- char * e = (char*)memchr(s, '\n', l);
- if (!e) e = (char*)memchr(s, '\r', l);
- if (!(s = strncasestr(s, "jtest", e - s)))
- return 0;
- s = (char*)memchr(s, '-', l);
- if (!s)
- return 0;
- s = (char*)memchr(s + 1, '-', l);
- if (!s)
- return 0;
- return ink_atoi(s + 1);
- }
- return 0;
-}
-
-static void make_response(int sock, int code) {
- fd[sock].response = fd[sock].req_header;
- fd[sock].length = sprintf( fd[sock].req_header, "%d\r\n", code);
- fd[sock].req_pos = 0;
- fd[sock].response_length = strlen(fd[sock].req_header);
- poll_set(sock, NULL, write_ftp_response);
-}
-
-static void make_long_response(int sock) {
- fd[sock].response = fd[sock].req_header;
- fd[sock].req_pos = 0;
- fd[sock].response_length = strlen(fd[sock].req_header);
- poll_set(sock, NULL, write_ftp_response);
-}
-
-static int send_ftp_data_when_ready(int sock) {
- if (fd[sock].state == STATE_FTP_DATA_READY && fd[sock].doc_length) {
- fd[sock].response = fd[sock].req_header;
- fd[sock].response_length = fd[sock].length = fd[sock].doc_length;
- if (verbose) printf("ftp data %d >-< %d\n", sock, fd[sock].ftp_data_fd);
- fd[sock].response = response_buffer + fd[sock].doc_length % 256;
- fd[sock].req_pos = 0;
- poll_set(sock, NULL, send_response);
- }
- return 0;
-}
-
-static int send_ftp_data(int sock, char * start, char * end) {
- int data_fd = fd[sock].ftp_data_fd;
- if (sscanf(start,"%d",&fd[data_fd].doc_length) != 1)
- return -1;
- fd[data_fd].doc = fd[sock].doc;
- send_ftp_data_when_ready(data_fd);
- return 0;
-}
-
-static int read_request(int sock) {
- if (verbose) printf("read_request %d\n", sock);
- int err = 0;
- int i;
-
- int maxleft = HEADER_SIZE - fd[sock].req_pos - 1;
-
- do {
- err = read (sock, &fd[sock].req_header[fd[sock].req_pos],
- maxleft);
- } while ((err < 0) && (errno == EINTR));
-
- if (err < 0) {
- if (errno == EAGAIN || errno == ENOTCONN) return 0;
- if (fd[sock].req_pos || errno != ECONNRESET)
- perror ("read");
- return -1;
- } else if (err == 0) {
- if (verbose) printf("eof\n");
- return -1;
- } else {
- if (verbose) printf("read %d got %d\n", sock, err);
- total_proxy_request_bytes += err;
- new_tbytes += err;
- fd[sock].req_pos += err;
- fd[sock].req_header[fd[sock].req_pos] = 0;
- char *buffer = fd[sock].req_header;
- for (i = fd[sock].req_pos - err;
- i < fd[sock].req_pos; i++)
- {
- switch (fd[sock].state) {
- case 0:
- if (buffer[i] == '\r')
- fd[sock].state = 1;
- else if (buffer[i] == '\n')
- fd[sock].state = 2;
- break;
- case 1:
- if (buffer[i] == '\n')
- fd[sock].state = 2;
- else
- fd[sock].state = 0;
- break;
- case 2:
- if (buffer[i] == '\r')
- fd[sock].state = 3;
- else if (buffer[i] == '\n') {
- fd[sock].state = 3;
- goto L3;
- } else
- fd[sock].state = 0;
- break;
- L3:
- case 3:
- if (buffer[i] == '\n') {
- if (show_headers)
- printf("Request from Proxy: {\n%s}\n",buffer);
- char host[80];
- int port, length;
- float r;
- if (sscanf(buffer,"GET http://%[^:]:%d/%f/%d",
- host,&port,&r,&length) == 4) {
- } else if (sscanf(buffer,"GET /%f/%d",&r,&length) == 2) {
- } else {
- if (verbose) printf("misscan: %s\n",buffer);
- fd[sock].close();
- return 0;
- }
- if (verbose)
- printf("read_request %d got request %d\n", sock, length);
- char * ims = strncasestr(buffer,"If-Modified-Since:", i);
- if (drand48() > ims_rate) ims = NULL;
- fd[sock].ims = ims?1:0;
- if (!ims) {
- fd[sock].response_length = fd[sock].length = length;
- fd[sock].nalternate = check_alt(fd[sock].req_header,
- strlen(fd[sock].req_header));
- fd[sock].response = response_buffer + length % 256 +
- fd[sock].nalternate;
- } else {
- fd[sock].nalternate = 0;
- if (verbose)
- printf("sending IMS 304: Not-Modified\n");
- fd[sock].response = NULL;
- fd[sock].response_length = fd[sock].length = 0;
- }
- fd[sock].req_pos = 0;
- if (!check_keepalive(fd[sock].req_header,
- strlen(fd[sock].req_header)))
- fd[sock].keepalive = 0;
- else
- fd[sock].keepalive--;
- if (fd[sock].length && drand48() < server_abort_rate) {
- fd[sock].length = (int)(drand48() * (fd[sock].length -1));
- fd[sock].keepalive = 0;
- }
- poll_set(sock,NULL,send_response);
- return 0;
- } else
- fd[sock].state = 0;
- break;
- }
- }
- }
- return 0;
-}
-
-static int send_compd_response(int sock) {
- int err = 0;
-
- struct {
- unsigned int code;
- unsigned int len;
- } compd_header;
- if (fd[sock].req_pos < sizeof(compd_header)) {
- compd_header.code = 0;
- compd_header.len = htonl((fd[sock].length * 2) / 3);
- do {
- err = write(sock, (char*)&compd_header + fd[sock].req_pos,
- sizeof(compd_header) - fd[sock].req_pos);
- } while ((err == -1) && (errno == EINTR));
- if (err <= 0) {
- if (!err) {
- if (verbose_errors) printf("write %d closed early\n", sock);
- goto Lerror;
- }
- if (errno == EAGAIN || errno == ENOTCONN) return 0;
- perror("write");
- goto Lerror;
- }
- if (verbose) printf("write %d %d\n", sock, err);
-
- new_tbytes += err;
- fd[sock].req_pos += err;
- fd[sock].bytes += err;
- fd[sock].response = response_buffer + (((fd[sock].length * 2) / 3) % 256);
- }
-
- if (fd[sock].req_pos < ((fd[sock].length * 2) / 3) + sizeof(compd_header)) {
- int towrite = cbuffersize;
- int desired = ((fd[sock].length * 2) / 3) + sizeof (compd_header) - fd[sock].req_pos;
- if (towrite > desired) {
- towrite = desired;
- }
- if (fast(sock,client_speed,fd[sock].bytes)) return 0;
- do {
- err = write (sock, fd[sock].response + fd[sock].req_pos - sizeof(compd_header) , towrite);
- } while ((err == -1) && (errno == EINTR));
- if (err < 0) {
- if (errno == EAGAIN || errno == ENOTCONN) return 0;
- fprintf(stderr,"write errno %d length %d sock %d\n",errno,towrite,sock);
- errors++;
- return -1;
- }
- if (verbose) printf("wrote %d %d\n", sock, err);
-
- new_tbytes += err;
- total_server_response_body_bytes += err;
- fd[sock].req_pos += err;
- fd[sock].bytes += err;
- }
-
- if (fd[sock].req_pos >= ((fd[sock].length * 2) / 3) + 4)
- return -1;
-
- return 0;
-Lerror:
- errors++;
- return 1;
-}
-
-static int read_compd_request(int sock) {
- if (verbose) printf("read_compd_request %d\n", sock);
- int err = 0;
- int i;
-
- if (fd[sock].req_pos < 4) {
- int maxleft = HEADER_SIZE - fd[sock].req_pos - 1;
- do {
- err = read (sock, &fd[sock].req_header[fd[sock].req_pos],
- maxleft);
- } while ((err < 0) && (errno == EINTR));
-
- if (err < 0) {
- if (errno == EAGAIN || errno == ENOTCONN) return 0;
- perror ("read");
- return -1;
- } else if (err == 0) {
- if (verbose) printf("eof\n");
- return -1;
- } else {
- if (verbose) printf("read %d got %d\n", sock, err);
- total_proxy_request_bytes += err;
- new_tbytes += err;
- fd[sock].req_pos += err;
- if (fd[sock].req_pos < 4)
- return 0;
- fd[sock].length = ntohl(*(unsigned int*)fd[sock].req_header);
- }
- }
-
- if (fd[sock].req_pos >= fd[sock].length + 4)
- goto Lcont;
-
- {
- char buf[MAX_BUFSIZE];
- int toread = cbuffersize;
- if (fast(sock,client_speed,fd[sock].bytes)) return 0;
- do {
- err = read(sock, buf, toread);
- } while ((err == -1) && (errno == EINTR));
- if (err < 0) {
- if (errno == EAGAIN || errno == ENOTCONN) return 0;
- if (errno == ECONNRESET) {
- if (verbose || verbose_errors)
- perror ("read");
- errors++;
- return -1;
- }
- panic_perror("read");
- }
- if (!err) {
- if (verbose || verbose_errors)
- perror ("read");
- errors++;
- return -1;
- }
- total_proxy_request_bytes += err;
- new_tbytes += err;
- fd[sock].req_pos += err;
- }
-
- if (fd[sock].req_pos >= fd[sock].length + 4)
- goto Lcont;
-
- return 0;
-
-Lcont:
- fd[sock].req_pos = 0;
- fd[sock].keepalive = 0;
- poll_set(sock,NULL,send_compd_response);
- return 0;
-}
-
-static int read_ftp_request(int sock) {
- if (verbose) printf("read_ftp_request %d\n", sock);
- int err = 0;
- int i;
-
- int maxleft = HEADER_SIZE - fd[sock].req_pos - 1;
-
- do {
- err = read (sock, &fd[sock].req_header[fd[sock].req_pos],
- maxleft);
- } while ((err < 0) && (errno == EINTR));
-
- if (err < 0) {
- if (errno == EAGAIN || errno == ENOTCONN) return 0;
- perror ("read");
- return -1;
- } else if (err == 0) {
- if (verbose) printf("eof\n");
- return -1;
- } else {
- if (verbose) printf("read %d got %d\n", sock, err);
- new_tbytes += err;
- fd[sock].req_pos += err;
- fd[sock].req_header[fd[sock].req_pos] = 0;
- char *buffer = fd[sock].req_header, *n;
- int res = 0;
- buffer[fd[sock].req_pos] = 0;
- if (verbose) printf("buffer [%s]\n", buffer);
-#define STREQ(_x,_s) (!strncasecmp(_x,_s,sizeof(_s)-1))
- if (STREQ(buffer,"USER")) {
- res = 331; goto Lhere;
- } else if (STREQ(buffer,"PASS")) {
- res = 230; goto Lhere;
- } else if (STREQ(buffer,"CWD")) {
- // TS used to send "CWD 1.2110000000..."
- // TS now sends "CWD /1.2110000000^M\n", so skip 5 instead of 4
- fd[sock].doc = (buffer[4]=='/') ? atof(buffer + 5) : atof(buffer + 4);
- res = 250; goto Lhere;
- } else if (STREQ(buffer,"TYPE")) {
- res = 200;
- Lhere:
- n = (char*)memchr(buffer,'\n',fd[sock].req_pos);
- if (!n) return 0;
- make_response(sock,res);
- return 0;
- } else if (STREQ(buffer,"SIZE")) {
- fd[sock].length =
- sprintf(fd[sock].req_header, "213 %d\r\n", atoi(buffer + 5));
- make_long_response(sock);
- return 0;
- } else if (STREQ(buffer,"MDTM")) {
- double err_rand = 1.0;
- if (ftp_mdtm_err_rate != 0.0) err_rand = drand48();
- if (err_rand < ftp_mdtm_err_rate) {
- fd[sock].length =
- sprintf (fd[sock].req_header, "550 mdtm file not found\r\n");
- } else {
- if (ftp_mdtm_rate == 0) {
- fd[sock].length =
- sprintf (fd[sock].req_header, "213 19900615100045\r\n");
- } else {
- time_t mdtm_now;
- time(&mdtm_now);
- if (mdtm_now-ftp_mdtm_last_update > ftp_mdtm_rate) {
- struct tm *mdtm_tm;
- ftp_mdtm_last_update = mdtm_now;
- mdtm_tm = localtime(&ftp_mdtm_last_update);
- sprintf(ftp_mdtm_str, "213 %.4d%.2d%.2d%.2d%.2d%.2d",
- mdtm_tm->tm_year + 1900,
- mdtm_tm->tm_mon + 1,
- mdtm_tm->tm_mday,
- mdtm_tm->tm_hour,
- mdtm_tm->tm_min,
- mdtm_tm->tm_sec);
- }
- fd[sock].length =
- sprintf (fd[sock].req_header, "%s\r\n", ftp_mdtm_str);
- }
- }
- make_long_response(sock);
- return 0;
- } else if (STREQ(buffer,"PASV")) {
- n = (char*)memchr(buffer,'\n',fd[sock].req_pos);
- if (!n) return 0;
- if ((fd[sock].ftp_data_fd = open_server(0, accept_ftp_data)) < 0)
- panic("could not open ftp data PASV accept port\n");
- fd[fd[sock].ftp_data_fd].ftp_data_fd = sock;
- if (verbose) printf("ftp PASV %d <-> %d\n", sock,fd[sock].ftp_data_fd);
- unsigned short p = fd[fd[sock].ftp_data_fd].name.sin_port;
- fd[sock].length =
- sprintf(fd[sock].req_header, "227 (%u,%u,%u,%u,%u,%u)\r\n",
- ((unsigned char*)&local_addr)[0],
- ((unsigned char*)&local_addr)[1],
- ((unsigned char*)&local_addr)[2],
- ((unsigned char*)&local_addr)[3],
- ((unsigned char*)&p)[0],
- ((unsigned char*)&p)[1]);
- if (verbose) puts(fd[sock].req_header);
- make_long_response(sock);
- fd[sock].ftp_mode = FTP_PASV;
- return 0;
- } else if (STREQ(buffer,"PORT")) {
- // watch out for an endian problems !!!
- char *start, *stop;
- for (start = buffer; !ParseRules::is_digit(*start); start++);
- for (stop = start; *stop != ','; stop++);
- for (i = 0; i < 4; i++) {
- ((unsigned char*)&(fd[sock].ftp_peer_addr))[i] =
- strtol(start, &stop, 10);
- for (start = ++stop; *stop != ','; stop++);
- }
- ((unsigned char*)&(fd[sock].ftp_peer_port))[0] =
- strtol(start, &stop, 10);
- start = ++stop;
- ((unsigned char*)&(fd[sock].ftp_peer_port))[1] =
- strtol(start, NULL, 10);
- fd[sock].length =
- sprintf(fd[sock].req_header, "200 Okay\r\n");
- if (verbose) puts(fd[sock].req_header);
- make_long_response(sock);
- fd[sock].ftp_mode = FTP_PORT;
- return 0;
- } else if (STREQ(buffer,"RETR")) {
- if (fd[sock].ftp_mode == FTP_NULL) {
- // default to PORT ftp
- struct sockaddr_in ftp_peer;
- int ftp_peer_addr_len = sizeof(ftp_peer);
- if (getpeername(sock, (struct sockaddr*)&ftp_peer,
-#if 0
- &ftp_peer_addr_len
-#else
- (socklen_t*)&ftp_peer_addr_len
-#endif
- ) < 0) {
- perror("getsockname");
- exit(EXIT_FAILURE);
- }
- fd[sock].ftp_peer_addr = ftp_peer.sin_addr.s_addr;
- fd[sock].ftp_peer_port = ftp_peer.sin_port;
- fd[sock].ftp_mode = FTP_PORT;
- }
- if (fd[sock].ftp_mode == FTP_PORT) {
- if ((fd[sock].ftp_data_fd =
- make_client(fd[sock].ftp_peer_addr,fd[sock].ftp_peer_port)) < 0)
- panic("could not open ftp PORT data connection to client\n");
- fd[fd[sock].ftp_data_fd].ftp_data_fd = sock;
- fd[fd[sock].ftp_data_fd].state = STATE_FTP_DATA_READY;
- if (verbose)
- printf("ftp PORT %d <-> %d\n", sock, fd[sock].ftp_data_fd);
- }
- n = (char*)memchr(buffer,'\n',fd[sock].req_pos);
- if (!n) return 0;
- if (send_ftp_data(sock, buffer+5, n)<0) {
- errors++;
- *n = 0;
- if (verbose)
- printf("badly formed ftp request: %s\n", buffer);
- return 1;
- }
- fd[sock].response = fd[sock].req_header;
- fd[sock].length = sprintf( fd[sock].req_header, "150 %d bytes\r\n",
- fd[fd[sock].ftp_data_fd].length);
- fd[sock].req_pos = 0;
- fd[sock].response_length = strlen(fd[sock].req_header);
- poll_set(sock, NULL, write_ftp_response);
- buffer = n+1;
- return 0;
- } else {
- if (verbose || verbose_errors) printf("ftp junk : %s\n", buffer);
- fd[sock].req_pos = 0;
- return 0;
- }
- }
-}
-
-static int accept_sock(int sock) {
- struct sockaddr_in clientname;
- int size = sizeof (clientname);
- int new_fd = 0;
- do {
- new_fd = accept(sock, (struct sockaddr *) &clientname,
-#if 0
- &size
-#else
- (socklen_t*)&size
-#endif
- );
- if (new_fd < 0) {
- if (errno == EAGAIN || errno == ENOTCONN) return 0;
- if (errno == EINTR || errno == ECONNABORTED) continue;
- printf("accept socket was %d\n",sock);
- panic_perror("accept");
- }
- } while (new_fd < 0);
-
- if (fcntl (new_fd, F_SETFL, O_NONBLOCK) < 0)
- panic_perror("fcntl");
-
-#if 0
-#ifdef BUFSIZE // make default
- int bufsize = BUFSIZE;
- if (setsockopt(new_fd,SOL_SOCKET,SO_SNDBUF,
- (const char *)&bufsize,sizeof(bufsize)) < 0) {
- perror("setsockopt");
- }
- if (setsockopt(new_fd,SOL_SOCKET,SO_SNDBUF,
- (const char *)&bufsize,sizeof(bufsize)) < 0) {
- perror("setsockopt");
- }
-#endif
-#endif
- int enable =1;
- if (setsockopt(new_fd,IPPROTO_TCP,TCP_NODELAY,
- (const char *)&enable,sizeof(enable)) < 0) {
- perror("setsockopt");
- }
-#ifdef PRINT_LOCAL_PORT
- struct sockaddr_in local_sa;
- size = sizeof(local_sa);
- getsockname(new_fd, (struct sockaddr*)&local_sa, &size);
- printf("local_sa.sin_port = %d\n", local_sa.sin_port);
-#endif
- return new_fd;
-}
-
-static int accept_compd (int sock) {
- int new_fd = accept_sock(sock);
- servers++;
- new_servers++;
- poll_init_set(new_fd, NULL, read_compd_request);
- fd[new_fd].count = &servers;
- fd[new_fd].start = now;
- fd[new_fd].ready = now + server_delay * HRTIME_MSECOND;
- fd[new_fd].keepalive = server_keepalive?server_keepalive:INT_MAX;
-
- return 0;
-}
-
-static int accept_read (int sock) {
- int new_fd = accept_sock(sock);
- servers++;
- new_servers++;
- if (ftp) {
- poll_init_set(new_fd, NULL, write_ftp_response);
- make_response(new_fd, 220);
- } else
- poll_init_set(new_fd, read_request);
- fd[new_fd].count = &servers;
- fd[new_fd].start = now;
- fd[new_fd].ready = now + server_delay * HRTIME_MSECOND;
- fd[new_fd].keepalive = server_keepalive?server_keepalive:INT_MAX;
-
- return 0;
-}
-
-static int accept_ftp_data (int sock) {
- int new_fd = accept_sock(sock);
- servers++;
- new_servers++;
- poll_init(new_fd);
- fd[new_fd].ftp_data_fd = fd[sock].ftp_data_fd;
- fd[fd[sock].ftp_data_fd].ftp_data_fd = new_fd;
- fd[new_fd].state = STATE_FTP_DATA_READY;
- fd[new_fd].count = &servers;
- fd[new_fd].start = now;
- fd[new_fd].ready = now + server_delay * HRTIME_MSECOND;
- fd[new_fd].keepalive = server_keepalive?server_keepalive:INT_MAX;
- fd[new_fd].state = STATE_FTP_DATA_READY;
- fd[new_fd].doc = fd[sock].doc;
- fd[new_fd].doc_length = fd[sock].doc_length;
- if (verbose)
- printf("accept_ftp_data %d for %d\n", new_fd, sock);
- send_ftp_data_when_ready(new_fd);
- return 1;
-}
-
-static int open_server(unsigned short int port, accept_fn_t accept_fn) {
- struct linger lngr;
- int sock;
- int one = 1;
- int err = 0;
-
- /* Create the socket. */
- sock = socket (AF_INET, SOCK_STREAM, 0);
- if (sock < 0) {
- perror ("socket");
- exit (EXIT_FAILURE);
- }
- struct sockaddr_in & name = fd[sock].name;
-
- /* Give the socket a name. */
- name.sin_family = AF_INET;
- name.sin_port = htons (port);
- name.sin_addr.s_addr = htonl (INADDR_ANY);
- if (setsockopt (sock, SOL_SOCKET, SO_REUSEADDR, (char *) &one,
- sizeof (one)) < 0) {
- perror((char*)"setsockopt");
- exit( EXIT_FAILURE);
- }
- if ((err=bind (sock, (struct sockaddr *) &name, sizeof (name))) < 0) {
- if (errno == EADDRINUSE)
- return -EADDRINUSE;
- perror("bind");
- exit( EXIT_FAILURE);
- }
-
- int addrlen = sizeof(name);
- if ((err = getsockname(sock, (struct sockaddr *) &name,
-#if 0
- &addrlen
-#else
- (socklen_t*)&addrlen
-#endif
- )) < 0) {
- perror("getsockname");
- exit( EXIT_FAILURE);
- }
- ink_assert(addrlen);
-
- /* Tell the socket not to linger on exit */
- lngr.l_onoff = 0;
- lngr.l_linger = 0;
- if (setsockopt (sock, SOL_SOCKET, SO_LINGER, (char*) &lngr,
- sizeof (struct linger)) < 0) {
- perror ("setsockopt");
- exit (EXIT_FAILURE);
- }
-
- if (listen (sock, 1024) < 0) {
- perror ("listen");
- exit (EXIT_FAILURE);
- }
-
- /* put the socket in non-blocking mode */
- if (fcntl (sock, F_SETFL, O_NONBLOCK) < 0) {
- perror ("fcntl");
- exit (EXIT_FAILURE);
- }
-
- if (verbose)
- printf("opening server on %d port %d\n", sock, name.sin_port);
-
- poll_init_set(sock, accept_fn);
-
- return sock;
-}
-
-// perform poll and invoke callbacks on active descriptors
-int poll_loop() {
- if (server_fd > 0) {
- while (read_ready(server_fd) > 0)
- accept_read(server_fd);
- }
- pollfd pfd[POLL_GROUP_SIZE];
- int ip = 0;
- int last_fd_in_block = 0;
- now = ink_get_hrtime();
- for (int i = 0 ; i <= last_fd ; i++) {
- if (fd[i].fd > 0 && (!fd[i].ready || now >= fd[i].ready)) {
- pfd[ip].fd = i;
- pfd[ip].events = 0;
- pfd[ip].revents = 0;
- if (fd[i].read_cb)
- pfd[ip].events |= POLLIN;
- if (fd[i].write_cb)
- pfd[ip].events |= POLLOUT;
- ip++;
- }
- if (ip >= POLL_GROUP_SIZE || i == last_fd) {
- int n = poll(pfd,ip,POLL_TIMEOUT);
- if (n > 0) {
- for (int j = 0; j < ip ; j++) {
- if (pfd[j].revents & (POLLIN|POLLERR|POLLHUP|POLLNVAL)) {
- if (verbose) printf("poll read %d %X\n",pfd[j].fd,pfd[j].revents);
- if (fd[pfd[j].fd].read_cb && fd[pfd[j].fd].read_cb(pfd[j].fd)) {
- fd[pfd[j].fd].close();
- continue;
- }
- }
- if (pfd[j].revents & (POLLOUT|POLLERR|POLLHUP|POLLNVAL)) {
- if (verbose) printf("poll write %d %X\n",pfd[j].fd,pfd[j].revents);
- if (fd[pfd[j].fd].write_cb && fd[pfd[j].fd].write_cb(pfd[j].fd)) {
- fd[pfd[j].fd].close();
- continue;
- }
- }
- }
- }
- ip = 0;
- }
- }
- return 0;
-}
-
-int gen_bfc_dist(double f = 10.0) {
-
- if (docsize >= 0) return docsize;
-
- double rand = 0.0;
- double rand2 = 0.0;
- bool f_given = f < 9.0;
- if (!f_given) {
- rand = drand48();
- rand2 = drand48();
- } else {
- rand = f;
- rand2 = (f * 13.0) - floor(f * 13.0);
- }
-
- int class_no;
- int file_no;
-
- if(rand < 0.35){
- class_no = 0;
- }else if(rand < 0.85){
- class_no = 1;
- }else if (rand < 0.99){
- class_no = 2;
- }else {
- class_no = 3;
- if (f_given) rand2 = (f * 113.0) - floor(f * 113.0);
- }
-
- if(rand2 < 0.018){
- file_no=0;
- }else if(rand2 < 0.091){
- file_no=1;
- }else if(rand2 < 0.237){
- file_no=2;
- }else if(rand2 < 0.432){
- file_no=3;
- }else if(rand2 < 0.627){
- file_no=4;
- }else if(rand2 < 0.783){
- file_no=5;
- }else if(rand2 < 0.887){
- file_no=6;
- }else if(rand2 < 0.945){
- file_no=7;
- }else if(rand2 < 1.000){
- file_no=8;
- }
- int size = 100;
- int i;
- for (i = 0; i < class_no; i++) {
- size = size * 10;
- }
- int increment = size;
- size = size * (file_no+1);
- // vary about the mean doc size for
- // that class/size
- if (!f_given)
- size += (int)((-increment * 0.5) + (increment * drand48()));
- if (verbose) printf("gen_bfc_dist %d\n",size);
- return size;
-}
-
-void build_response() {
- int maxsize = docsize > MAX_RESPONSE_LENGTH ? docsize : MAX_RESPONSE_LENGTH;
- response_buffer = (char*) malloc (maxsize + HEADER_SIZE);
- for (int i = 0 ; i < maxsize + HEADER_SIZE; i++)
- response_buffer[i] = i % 256;
-}
-
-static void put_ka(int sock) {
- int i = 0;
- for (; i < n_ka_cache; i++ )
- if (!ka_cache_head[i] || fd[ka_cache_head[i]].ip == fd[sock].ip)
- goto Lpush;
- i = n_ka_cache++;
-Lpush:
- if (ka_cache_tail[i])
- fd[ka_cache_tail[i]].next = sock;
- else
- ka_cache_head[i] = sock;
- ka_cache_tail[i] = sock;
-}
-
-static int get_ka(unsigned int ip) {
- for (int i = 0; i < n_ka_cache; i++ )
- if (fd[ka_cache_head[i]].ip == ip) {
- int res = ka_cache_head[i];
- ka_cache_head[i] = fd[ka_cache_head[i]].next;
- if (res == ka_cache_tail[i]) {
- ink_assert(!ka_cache_head[i]);
- ka_cache_tail[i] = 0;
- }
- return res;
- }
- return -1;
-}
-
-static void defer_url(char * url) {
- if (n_defered_urls < MAX_DEFERED_URLS -1)
- defered_urls[n_defered_urls++] = strdup(url);
- else
- fprintf(stderr, "too many defered urls, dropping '%s'\n", url);
-}
-
-static int throttling_connections() {
- return client_rate && keepalive_cons && current_clients >= keepalive_cons;
-}
-
-static void done() {
- interval_report();
- exit(0);
-}
-
-static int is_done() {
- return
- (urls_mode && !current_clients && !n_defered_urls) ||
- (bandwidth_test && bandwidth_test_to_go <= 0 && !current_clients);
-}
-
-static void undefer_url(bool unthrottled) {
- if ((unthrottled || !throttling_connections()) && n_defered_urls) {
- --n_defered_urls;
- char * url = defered_urls[n_defered_urls];
- make_url_client(url,0,true,unthrottled);
- free(url);
- if (verbose)
- printf("undefer_url: made client %d clients\n", current_clients);
- } else
- if (verbose) printf("undefer_url: throttle\n");
- if (is_done())
- done();
-}
-
-static void init_client(int sock) {
- poll_init(sock);
- fd[sock].start = now;
- fd[sock].ready = now;
- fd[sock].count = &clients;
- poll_set(sock,NULL,write_request);
-}
-
-static unsigned int get_addr(char * host) {
- unsigned int addr = inet_addr(host);
- struct hostent *host_info = NULL;
-
- if (!addr || (-1 == (int)addr)) {
- host_info = gethostbyname(host);
- if (!host_info) {
- perror ("gethostbyname");
- return (unsigned int)-1;
- }
- addr = *((unsigned int*) host_info->h_addr);
- }
-
- return addr;
-}
-
-char * find_href_end (char *start, int len)
-{
- char *end = start;
- if (!start) return NULL;
-
- while (*end && len > 0) {
- if (*end == '\"') break; /* " */
- if (*end == '\'') break;
- if (*end == '>') break;
- if (*end == ' ') break;
- if (*end == '\t') break;
- if (*end == '\n') break;
- if (*end == '<') break;
- if(*end & 0x80) break; /* hi order bit! */
- len--;
- end++;
- }
-
- if (*end == 0 || len == 0)
- return NULL;
- else
- return end;
-} // find_href_end
-
-char * find_href_start(char * tag, char *base, int len)
-{
- int taglen = strlen(tag);
- if (base == NULL) return NULL;
-
- char *start = base;
- char *end = base + len;
-
- Lagain:
- {
- start = strncasestr(start, tag, len);
- if ((start == NULL) || (end-start < 6)) {
- return NULL;
- }
- start += taglen;
- len -= taglen;
- } // block
-
- while (ParseRules::is_ws (*start) && (end - start > 1)) {
- start++;
- len--;
- }
- if (*start == '=' && (end - start > 1)) {
- start++;
- len--;
- } else {
- goto Lagain;
- }
- while (ParseRules::is_ws(*start) && (end - start > 1)) {
- start++;
- len--;
- }
- //
- // Optional quotes: href="x" or href='x' or href=x
- //
- if ((*start == '\"'||(*start == '\'')) && (end - start > 1)) { /*"'*/
- start++;
- len--;
- }
- while (ParseRules::is_ws(*start) && (end - start > 1)) {
- start++;
- len--;
- }
-
- return start;
-} // find_href_start
-
-int compose_url(char * new_url, char * base, char *input) {
- char sche[8],host[512],port[10],path[512],frag[512],quer[512],para[512];
- char curl[512];
- int xsche,xhost,xport,xpath,xfrag,xquer,xpar,rel,slash;
- ink_web_decompose_url(base,sche,host,port,path,frag,quer,para,
- &xsche,&xhost,&xport,&xpath,&xfrag,&xquer,
- &xpar,&rel,&slash);
- strcpy(curl, "http://");
- strcat(curl,host);
- if (xport) {
- strcat(curl,":");
- strcat(curl,port);
- }
- strcat(curl,"/");
- strcat(curl,path);
-
- ink_web_canonicalize_url(curl, input, new_url, 512);
- return 0;
-} // compose_urls
-
-void compose_all_urls( char * tag, char * buf, char * start, char * end,
- int buflen, char * base_url)
-{
- char old;
- while ((start = find_href_start(tag, end, buflen - (end - buf)))) {
- char newurl[512];
- end = (char *) find_href_end(start, min(buflen - (start-buf), 512 - 10));
- if (!end) {
- end = start + strlen(tag);
- continue;
- }
- old = *end;
- *end = 0;
- compose_url(newurl, base_url, start);
- make_url_client(newurl,base_url);
- *end = old;
- } // while
-}
-//
-// Input is a NULL-terminated string (buf of buflen)
-// also, a read-write base_url
-//
-void extract_urls(char * buf, int buflen, char * base_url) {
- //if (verbose) printf("EXTRACT<<%s\n>>", buf);
- char old;
- char *start = NULL;
- char *end = NULL;
- char old_base[512];
- strcpy(old_base, base_url);
-
- start = strncasestr(buf, "<base ", buflen);
- if (start) {
- end = (char*)memchr(start, '>', buflen - (start - buf));
- if (end) {
- char * rover = strncasestr(start, "href", end - start);
- if (rover) {
- rover += 4;
- while (rover < end &&
- (ParseRules::is_ws(*rover) || *rover == '=' || *rover == '\''
- || *rover == '\"')) /* " */
- rover++;
- start = rover;
- while (rover < end &&
- !(ParseRules::is_ws(*rover) || *rover == '\''
- || *rover == '\"'))
- rover++;
- old = *rover;
- *rover = 0;
- compose_url(base_url,old_base,start);
- // fixup unqualified hostnames (e.g. http://internal/foo)
- char * he = strchr(base_url + 8, '/');
- if (!memchr(base_url,'.',he-base_url)) {
- char t[512]; strcpy(t,base_url);
- char * old_he = strchr(old_base + 8, '.');
- if (old_he) {
- char * old_hee = strchr(old_he, '/');
- if (old_hee) {
- memcpy(base_url,t,(he-base_url));
- memcpy(base_url + (he-base_url), old_he, (old_hee-old_he));
- memcpy(base_url + (he-base_url) + (old_hee-old_he),
- t+(he-base_url), strlen(t+(he-base_url)));
- base_url[(he-base_url) + (old_hee-old_he) +
- strlen(t+(he-base_url))] = 0;
- }}}}}}
-
- end = buf;
- if (follow)
- compose_all_urls("href", buf, start, end, buflen, base_url);
- if (fullpage) {
- char *tags[] = { "src", "image", "object", "archive", "background",
- // "location", "code"
- };
- for (int i = 0 ; i < sizeof(tags)/sizeof(tags[0]) ; i++)
- compose_all_urls(tags[i], buf, start, end, buflen, base_url);
- }
-} // extract_urls
-
-void follow_links(int sock) {
- if (urls_mode) {
- if (fd[sock].binary) return;
- int l = fd[sock].response_remaining;
- char * r = fd[sock].response, *p = r, *n = r;
- if (r)
- extract_urls(r,l,fd[sock].base_url);
- if (l < MAX_BUFSIZE) {
- while (n) {
- n = (char*)memchr(p,'\n',l - (p - r));
- if (!n) n = (char*)memchr(p,'\r',l - (p - r));
- if (n) p = n + 1;
- }
- int done = p - r, remaining = l - done;
- if (done) {
- memmove(r, p, remaining);
- fd[sock].response_remaining = remaining;
- }
- } else // bail
- fd[sock].response_length = 0;
- }
-}
-
-static int verify_content(int sock, char * buf, int done) {
- if (urls_mode && !check_content)
- return 1;
- int l = fd[sock].response_length;
- char * d = response_buffer + (l % 256) + fd[sock].nalternate;
- int left = fd[sock].length;
- if (left > 0) {
- if (embed_url && !fd[sock].jg_compressed) {
- if (l == left && left > 64) {
- char *url_end = NULL, *url_start = NULL;
- get_path_from_req(fd[sock].base_url,&url_start,&url_end);
- if (url_end - url_start < done) {
- if (memcmp(url_start,buf,url_end - url_start))
- return 0;
- }
- }
- // skip past the URL which is embedded in the document
- // to confound the fingerprinting code
- if (l - left < 64) {
- int skip = 64 - (l - left);
- left -= skip;
- done -= skip;
- buf += skip;
- if (done < 0)
- done = 0;
- }
- }
- if (!check_content)
- return 1;
- if (done > left)
- done = left;
- if (memcmp(buf, d + (fd[sock].response_length - left), done))
- return 0;
- }
- return 1;
-}
-
-#define ZIPF_SIZE (1<<20)
-static double *zipf_table = NULL;
-void build_zipf() {
- zipf_table = (double*)malloc(ZIPF_SIZE * sizeof(double));
- for (int i = 0; i < ZIPF_SIZE; i++)
- zipf_table[i] = 1.0 / pow(i+2, zipf);
- for (int i = 1; i < ZIPF_SIZE; i++)
- zipf_table[i] = zipf_table[i-1] + zipf_table[i];
- double x = zipf_table[ZIPF_SIZE-1];
- for (int i = 0; i < ZIPF_SIZE; i++)
- zipf_table[i] = zipf_table[i] / x;
-}
-
-int get_zipf(double v) {
- int l = 0, r = ZIPF_SIZE-1, m;
- do {
- m = (r + l) / 2;
- if (v < zipf_table[m])
- r = m - 1;
- else
- l = m + 1;
- } while (l < r);
- if (zipf_bucket_size == 1)
- return m;
- double x = zipf_table[m], y = zipf_table[m+1];
- m += (v - x) / (y - x);
- return m;
-}
-
-static int read_response_error(int sock) {
- errors++;
- fd[sock].close();
- if (!urls_mode)
- make_bfc_client(proxy_addr, proxy_port);
- return 0;
-}
-
-static int read_for_error(int sock) {
- char b[8192];
- int err = read(sock, &b, 8192);
- if (err < 0)
- return read_response_error(sock);
- return 0;
-}
-
-static int read_response(int sock) {
- int err = 0;
-
- if (fd[sock].req_pos >= 0) {
- if (!fd[sock].req_pos)
- memset(fd[sock].req_header,0,HEADER_SIZE);
- do {
- int l = HEADER_SIZE - fd[sock].req_pos - 1;
- if (l <= 0) {
- if (verbose || verbose_errors)
- printf("header too long '%s'", fd[sock].req_header);
- return read_response_error(sock);
- }
- err = read(sock, fd[sock].req_header + fd[sock].req_pos,
- HEADER_SIZE - fd[sock].req_pos - 1);
- } while ((err == -1) && (errno == EINTR));
- if (err <= 0) {
- if (!err) {
- if (verbose_errors)
- printf("read_response %d closed during header for '%s' after %d%s\n",
- sock, fd[sock].base_url, fd[sock].req_pos,
- (keepalive && (fd[sock].keepalive != keepalive)
- && !fd[sock].req_pos) ? " -- keepalive timeout" : "");
- return read_response_error(sock);
- }
- if (errno == EAGAIN || errno == ENOTCONN) return 0;
- if (errno == ECONNRESET) {
- if (!fd[sock].req_pos && keepalive > 0 &&
- fd[sock].keepalive != keepalive) {
- fd[sock].close();
- if (!urls_mode)
- make_bfc_client(proxy_addr, proxy_port);
- return 0;
- }
- if (verbose || verbose_errors)
- perror ("read");
- goto Ldone;
- }
- panic_perror("read");
- }
- if (verbose)
- printf("read %d header %d [%s]\n",
- sock, err, fd[sock].req_header);
- b1_ops++;
-
- strcpy(fd[sock].response_header, fd[sock].req_header);
-
- b1latency += ((ink_get_hrtime() - fd[sock].start) / HRTIME_MSECOND);
- new_cbytes += err;
- new_tbytes += err;
- fd[sock].req_pos += err;
- fd[sock].bytes += err;
- fd[sock].active = ink_get_hrtime();
- int total_read = fd[sock].req_pos;
- char * p = fd[sock].req_header;
- char * cl = NULL;
- int cli = 0;
- while ((p = strchr(p,'\n'))) {
- if (verbose) printf("read header end? [%s]\n", p);
- if (p[1] == '\n' || (p[1] == '\r' && p[2] == '\n')) {
- int off = 1 + (p[1] == '\r' ? 2 : 1);
- p += off;
- strncpy(fd[sock].response_header, fd[sock].req_header, p - fd[sock].req_header);
- fd[sock].response_header[p - fd[sock].req_header] = '\0';
- int lbody = fd[sock].req_pos - (p - fd[sock].req_header);
- cl = strncasestr(fd[sock].req_header,"Content-Length:",
- p - fd[sock].req_header);
- if (cl) {
- cli = atoi(cl + 16);
- int expected_length = fd[sock].response_length;
- if (compd_suite) {
- if (strstr(fd[sock].req_header, "x-jg")) {
- fd[sock].jg_compressed = 1;
- expected_length = (fd[sock].response_length * 2) / 3;
- }
- }
- if (fd[sock].response_length && verbose_errors &&
- expected_length != cli && !nocheck_length)
- fprintf(stderr, "bad Content-Length expected %d got %d orig %d",
- expected_length, cli, fd[sock].response_length);
- fd[sock].response_length = fd[sock].length = cli;
- }
- if (fd[sock].req_header[9] == '2') {
- if (!verify_content(sock,p,lbody)) {
- if (verbose || verbose_errors)
- printf("content verification error '%s'\n", fd[sock].base_url);
- return read_response_error(sock);
- }
- }
- total_proxy_response_body_bytes += lbody;
- total_proxy_response_header_bytes += p - fd[sock].req_header;
- fd[sock].length -= lbody;
- fd[sock].req_pos = -1;
- if (fd[sock].length && drand48() < client_abort_rate) {
- fd[sock].client_abort = 1;
- fd[sock].length = (int)(drand48() * (fd[sock].length -1));
- fd[sock].keepalive = 0;
- fd[sock].drop_after_CL = 1;
- }
- if (verbose) printf("read %d header done\n", sock);
- break;
- }
- p++;
- }
- if (!p) return 0;
- int hlen = p - fd[sock].req_header;
- if (show_headers) {
- printf("Response From Proxy: {\n");
- for (char * c = fd[sock].req_header; c < p ; c++)
- putc(*c,stdout);
- printf("}\n");
- }
- if (obey_redirects && urls_mode &&
- fd[sock].req_header[9] == '3' &&
- fd[sock].req_header[10] == '0' &&
- (fd[sock].req_header[11] == '1' || fd[sock].req_header[11] == '2'))
- {
- char * redirect = strstr(fd[sock].req_header,"http://");
- char * e = redirect?(char*)memchr(redirect,'\n',hlen):0;
- if (!redirect || !e)
- fprintf(stderr, "bad redirect '%s'",fd[sock].req_header);
- else {
- if (e[-1]=='\r') e--;
- *e = 0;
- make_url_client(redirect);
- }
- fd[sock].close();
- return 0;
- }
- if (fd[sock].req_header[9] != '2') {
- if (verbose_errors) {
- char * e = (char*)memchr(fd[sock].req_header, '\r', hlen);
- if (e) *e = 0;
- else {
- char * e = (char*)memchr(fd[sock].req_header, '\n', hlen);
- if (e) *e = 0;
- else *p = 0;
- }
- printf("error response %d: '%s':'%s'\n", sock,
- fd[sock].base_url, fd[sock].req_header);
- }
- return read_response_error(sock);
- }
- char * r = fd[sock].req_header;
- int length = p - r;
- char * ka = check_keepalive(r, length);
- if (urls_mode) {
- fd[sock].response_remaining = total_read - length;
- if (fd[sock].response_remaining)
- memcpy(fd[sock].response,p,fd[sock].response_remaining);
- if (check_content && !cl) {
- if (verbose || verbose_errors)
- printf("missiing Content-Length '%s'\n", fd[sock].base_url);
- return read_response_error(sock);
- }
- } else
- fd[sock].response = 0;
- if (!cl || !ka)
- fd[sock].keepalive = -1;
- if (!cl)
- fd[sock].length = INT_MAX;
- }
-
- if (fd[sock].length <= 0 &&
- (fd[sock].keepalive > 0 || fd[sock].drop_after_CL))
- goto Ldone;
-
- {
- char * r = NULL;
- char buf[MAX_BUFSIZE];
- int toread = cbuffersize;
- if (urls_mode) {
- if (fd[sock].response_remaining + cbuffersize < MAX_BUFSIZE)
- r = fd[sock].response + fd[sock].response_remaining;
- else {
- toread = MAX_BUFSIZE - fd[sock].response_remaining;
- if (!toread) {
- if (verbose_errors || verbose)
- fprintf(stderr,"line exceeds buffer, unable to follow links\n");
- toread = cbuffersize;
- r = fd[sock].response;
- fd[sock].response_remaining = 0;
- } else
- r = fd[sock].response + fd[sock].response_remaining;
- }
- } else
- r = buf;
- if (fast(sock,client_speed,fd[sock].bytes)) return 0;
- if (fd[sock].bytes > abort_retry_bytes &&
- (((now - fd[sock].start + 1)/HRTIME_SECOND) > abort_retry_secs) &&
- !faster_than(sock,abort_retry_speed,fd[sock].bytes))
- {
- fd[sock].client_abort = 1;
- fd[sock].keepalive = 0;
- if (!urls_mode && !client_rate)
- make_bfc_client(proxy_addr, proxy_port);
- goto Ldone;
- }
- do {
- err = read(sock, r, toread);
- } while ((err == -1) && (errno == EINTR));
- if (err < 0) {
- if (errno == EAGAIN || errno == ENOTCONN) return 0;
- if (errno == ECONNRESET) {
- if (verbose || verbose_errors)
- perror ("read");
- goto Ldone;
- }
- panic_perror("read");
- }
- if (!err)
- goto Ldone;
- if (!verify_content(sock,buf,err)) {
- if (verbose || verbose_errors)
- printf("content verification error '%s'\n", fd[sock].base_url);
- return read_response_error(sock);
- }
- total_proxy_response_body_bytes += err;
- new_cbytes += err;
- new_tbytes += err;
- fd[sock].response_remaining += err;
- fd[sock].bytes += err;
- follow_links(sock);
- if (fd[sock].length != INT_MAX)
- fd[sock].length -= err;
- fd[sock].active = ink_get_hrtime();
- if (verbose)
- printf("read %d got %d togo %d %d %d\n", sock, err, fd[sock].length,
- fd[sock].keepalive, fd[sock].drop_after_CL);
- }
-
- if (fd[sock].length <= 0 &&
- (fd[sock].keepalive > 0 || fd[sock].drop_after_CL))
- goto Ldone;
-
- return 0;
-
-Ldone:
- int ok = false;
- if (!fd[sock].client_abort &&
- !(server_abort_rate > 0) &&
- fd[sock].length && fd[sock].length != INT_MAX)
- {
- if (verbose || verbose_errors)
- printf("bad length %d wanted %d after %d ms: '%s'\n",
- fd[sock].response_length - fd[sock].length,
- fd[sock].response_length,
- (int)((ink_get_hrtime() - fd[sock].active)/HRTIME_MSECOND),
- fd[sock].base_url);
- return read_response_error(sock);
- }
- if (verbose) printf("read %d done\n", sock);
- new_ops++;
- double thislatency =((ink_get_hrtime() - fd[sock].start) / HRTIME_MSECOND);
- latency += (int)thislatency;
- lat_ops++;
- if (fd[sock].keepalive > 0) {
- fd[sock].reset();
- put_ka(sock);
- current_clients--;
- if (urls_mode) {
- undefer_url();
- return 0;
- }
- } else
- fd[sock].close();
- if (!urls_mode && !client_rate)
- make_bfc_client(proxy_addr, proxy_port);
- return 0;
-}
-
-static int write_request(int sock) {
- int err = 0;
-
- do {
- err = write(sock, fd[sock].req_header + fd[sock].req_pos,
- fd[sock].length - fd[sock].req_pos);
- } while ((err == -1) && (errno == EINTR));
- if (err <= 0) {
- if (!err) {
- if (verbose_errors) printf("write %d closed early\n", sock);
- goto Lerror;
- }
- if (errno == EAGAIN || errno == ENOTCONN) return 0;
- perror("write");
- goto Lerror;
- }
- if (verbose) printf("write %d %d\n", sock, err);
-
- new_tbytes += err;
- total_client_request_bytes += err;
- fd[sock].req_pos += err;
- fd[sock].active = ink_get_hrtime();
-
- if (fd[sock].req_pos >= fd[sock].length) {
- if (verbose) printf("write complete %d %d\n", sock, fd[sock].length);
- fd[sock].req_pos = 0;
- fd[sock].length = fd[sock].response_length;
- poll_set(sock, read_response);
- }
- return 0;
-Lerror:
- errors++;
-#ifndef RETRY_CLIENT_WRITE_ERRORS
- if (!--nclients)
- panic("no more clients\n");
- return 1;
-#else
- if (!urls_mode)
- make_bfc_client(proxy_host, proxy_port);
- fd[sock].close();
- return 0;
-#endif
-}
-
-static int write_ftp_response(int sock) {
- int err = 0;
-
- do {
- err = write(sock, fd[sock].req_header + fd[sock].req_pos,
- fd[sock].length - fd[sock].req_pos);
- } while ((err == -1) && (errno == EINTR));
-
- if (err <= 0) {
- if (!err) {
- if (verbose_errors) printf("write %d closed early\n", sock);
- goto Lerror;
- }
- if (errno == EAGAIN || errno == ENOTCONN) return 0;
- perror("write");
- goto Lerror;
- }
- if (verbose) printf("write %d %d\n", sock, err);
-
- new_tbytes += err;
- fd[sock].req_pos += err;
-
- if (fd[sock].req_pos >= fd[sock].length) {
- if (verbose) printf("write complete %d %d\n", sock, fd[sock].length);
- fd[sock].req_pos = 0;
- fd[sock].length = fd[sock].response_length;
- poll_set(sock, read_ftp_request);
- }
- return 0;
-Lerror:
- errors++;
- return 1;
-}
-
-static int make_client (unsigned int addr, int port) {
- struct linger lngr;
-
- int sock = socket (PF_INET, SOCK_STREAM, 0);
- if (sock < 0)
- panic_perror ("socket");
-
- if (fcntl (sock, F_SETFL, O_NONBLOCK) < 0)
- panic_perror ("fcntl");
-
- /* tweak buffer size so that remote end can't close connection too fast */
-
-#if 0
- int bufsize = cbuffersize;
- if (setsockopt(sock,SOL_SOCKET,SO_RCVBUF,
- (const char *)&bufsize,sizeof(bufsize)) < 0)
- panic_perror("setsockopt");
- if (setsockopt(sock,SOL_SOCKET,SO_SNDBUF,
- (const char *)&bufsize,sizeof(bufsize)) < 0)
- panic_perror("setsockopt");
-#endif
- int enable =1;
- if (setsockopt(sock,IPPROTO_TCP,TCP_NODELAY,
- (const char *)&enable,sizeof(enable)) < 0)
- panic_perror("setsockopt");
-
- /* Tell the socket not to linger on exit */
- lngr.l_onoff = 1;
- lngr.l_linger = 0;
- if (!ftp) { // this causes problems for PORT ftp -- ewong
- if (setsockopt (sock, SOL_SOCKET, SO_LINGER, (char*) &lngr,
- sizeof (struct linger)) < 0) {
- perror ("setsockopt");
- exit (EXIT_FAILURE);
- }
- }
-
- /* Give the socket a name. */
- struct sockaddr_in name;
- name.sin_family = AF_INET;
- name.sin_port = htons(port);
- name.sin_addr.s_addr = addr;
-
- if (verbose) printf("connecting to %u.%u.%u.%u:%d\n",
- ((unsigned char*)&addr)[0], ((unsigned char*)&addr)[1],
- ((unsigned char*)&addr)[2], ((unsigned char*)&addr)[3],
- port);
-
- while (connect (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
- if (errno == EINTR) continue;
- if (errno == EINPROGRESS)
- break;
- if (verbose_errors) fprintf(stderr, "connect failed errno = %d\n", errno);
- errors++;
- return -1;
- }
-
- init_client(sock);
- fd[sock].ip = addr;
- clients++;
- current_clients++;
- new_clients++;
- return sock;
-}
-
-static void make_bfc_client (unsigned int addr, int port) {
- int sock = -1;
- if (bandwidth_test && bandwidth_test_to_go-- <= 0)
- return;
- if (keepalive)
- sock = get_ka(addr);
- if (sock < 0) {
- sock = make_client(addr,port);
- fd[sock].keepalive = keepalive;
- } else {
- init_client(sock);
- current_clients++;
- fd[sock].keepalive--;
- }
- if (sock<0)
- panic("unable to open client connection\n");
- double h = drand48();
- double dr = drand48();
- if (zipf == 0.0) {
- if (h < hitrate) {
- dr = 1.0 + (floor(dr * hotset) / hotset);
- fd[sock].response_length = gen_bfc_dist(dr - 1.0);
- } else
- fd[sock].response_length = gen_bfc_dist(dr);
- } else {
- unsigned long long int doc = get_zipf(dr);
- // Some large randomish number.
- unsigned long long int doc_len_int = doc * 0x14A4D0FB0E93E3A7LL;
- unsigned long int x = doc_len_int;
- double y = (double)x;
- y /= 0x100000000LL; // deterministic random number between 0 and 1.0
- fd[sock].response_length = gen_bfc_dist(y);
- dr = doc;
- }
- if (verbose) printf("gen_bfc_dist %d\n", fd[sock].response_length);
- char eheaders[16384];
- *eheaders = 0;
- int nheaders = extra_headers;
- if (nheaders>0) {
- char * eh = eheaders;
- if (!vary_user_agent) {
- eh += sprintf(eh, "User-Agent: Mozilla/4.04 [en] (X11; I; Linux 2.0.31 i586)\r\n");
- nheaders--;
- }
- if (nheaders>0)
- eh += sprintf(eh, "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, image/png, */*\r\n");
- while (--nheaders>0)
- eh += sprintf(eh, "Extra-Header%d: a lot of junk for header %d\r\n",
- nheaders, nheaders);
- }
- char cookie[256];
- *cookie = 0;
- fd[sock].nalternate = (int)(alternates * drand48());
- if (alternates)
- if (!vary_user_agent)
- sprintf(cookie, "Cookie: jtest-cookie-%d\r\n", fd[sock].nalternate);
- else
- sprintf(cookie, "User-Agent: jtest-browser-%d\r\n", fd[sock].nalternate);
- char* extension;
- switch (request_extension) {
- case 1: extension = (char*)".html"; break;
- case 2: extension = (char*)".jpeg"; break;
- case 3: extension = (char*)"/"; break;
- default:
- extension = (char*)(compd_suite ? ".jpeg" : "");
- }
- char evo_str[20];
- evo_str[0] = '\0';
- if (evo_rate != 0.0) {
- double evo_index = dr + (((double)now)/HRTIME_HOUR) * evo_rate;
- sprintf(evo_str, ".%u", ((unsigned int)evo_index));
- }
- if (0 == hostrequest) {
- sprintf(fd[sock].req_header,
- ftp ?
- "GET ftp://%s:%d/%12.10f/%d%s%s HTTP/1.0\r\n"
- "%s"
- "%s"
- "%s"
- "%s"
- "\r\n"
- :
- "GET http://%s:%d/%12.10f/%d%s%s HTTP/1.0\r\n"
- "%s"
- "%s"
- "%s"
- "%s"
- "\r\n"
- ,
- local_host, server_port, dr,
- fd[sock].response_length, evo_str, extension,
- fd[sock].keepalive?"Proxy-Connection: Keep-Alive\r\n":"",
- reload_rate > drand48() ? "Pragma: no-cache\r\n":"",
- eheaders, cookie
- );
- } else if (1 == hostrequest) {
- sprintf(fd[sock].req_header,
- "GET /%12.10f/%d%s%s HTTP/1.0\r\n"
- "Host: %s:%d\r\n"
- "%s"
- "%s"
- "%s"
- "%s"
- "\r\n",
- dr, fd[sock].response_length, evo_str, extension,
- local_host, server_port,
- fd[sock].keepalive?"Connection: Keep-Alive\r\n":"",
- reload_rate > drand48() ? "Pragma: no-cache\r\n":"",
- eheaders, cookie);
- } else if (2 == hostrequest) {
- /* Send a non-proxy client request i.e. for Transparency testing */
- sprintf(fd[sock].req_header,
- "GET /%12.10f/%d%s%s HTTP/1.0\r\n"
- "%s"
- "%s"
- "%s"
- "%s"
- "\r\n",
- dr, fd[sock].response_length, evo_str, extension,
- fd[sock].keepalive?"Connection: Keep-Alive\r\n":"",
- reload_rate > drand48() ? "Pragma: no-cache\r\n":"",
- eheaders,
- cookie);
- }
- if (verbose) printf("request %d [%s]\n", sock, fd[sock].req_header);
- fd[sock].length = strlen(fd[sock].req_header);
- {
- char * s = fd[sock].req_header;
- char * e = (char*)memchr(s, '\r', 512);
- char * url = fd[sock].base_url;
- memcpy(url,s,e-s);
- url[e-s] = 0;
- if (show_before) printf("%s\n", url);
- }
- if (show_headers) printf("Request to Proxy: {\n%s}\n", fd[sock].req_header);
-}
-
-#define RUNNING(_n) \
- total_##_n = (((total_##_n * (average_over-1))/average_over) + new_##_n); \
- running_##_n = total_##_n / average_over; \
- new_##_n = 0;
-
-#define RUNNING_AVG(_t,_n,_o) \
- _t = _o?((_t * (average_over-1) + _n/_o) / average_over):_t; \
- _n = 0;
-
-void interval_report() {
- static int here = 0;
- now = ink_get_hrtime();
- if (!(here++ % 20))
- printf(
- " con new ops 1B lat bytes/per svrs new ops total time err\n");
- RUNNING(clients);
- RUNNING_AVG(running_latency,latency,lat_ops); lat_ops = 0;
- RUNNING_AVG(running_b1latency,b1latency,b1_ops); b1_ops = 0;
- RUNNING(cbytes);
- RUNNING(ops);
- RUNNING(servers);
- RUNNING(sops);
- RUNNING(tbytes);
- float t = (float)(now - start_time);
- uint64_t per = current_clients ? running_cbytes / current_clients : 0;
- printf("%4d %4d %7.1f %4d %4d %10" PRIu64"/%-6" PRIu64" %4d %4d %4d %9" PRIu64" %6.1f %4d\n",
- current_clients, // clients, n_ka_cache,
- running_clients,
- running_ops, running_b1latency, running_latency,
- running_cbytes, per,
- running_servers,
- running_servers,
- running_sops, running_tbytes,
- t/((float)HRTIME_SECOND),
- errors);
- if (is_done()) {
- printf("Total Client Request Bytes:\t\t%" PRIu64"\n", total_client_request_bytes);
- printf("Total Server Response Header Bytes:\t%" PRIu64"\n",
- total_server_response_header_bytes);
- printf("Total Server Response Body Bytes:\t%" PRIu64"\n",
- total_server_response_body_bytes);
- printf("Total Proxy Request Bytes:\t\t%" PRIu64"\n", total_proxy_request_bytes);
- printf("Total Proxy Response Header Bytes:\t%" PRIu64"\n",
- total_proxy_response_header_bytes);
- printf("Total Proxy Response Body Bytes:\t%" PRIu64"\n",
- total_proxy_response_body_bytes);
- }
-}
-
-#define URL_HASH_ENTRIES url_hash_entries
-#define BYTES_PER_ENTRY 3
-#define ENTRIES_PER_BUCKET 16
-#define OVERFLOW_ENTRIES 1024 // many many
-
-#define BUCKETS (URL_HASH_ENTRIES / ENTRIES_PER_BUCKET)
-#define BYTES_PER_BUCKET (BYTES_PER_ENTRY * ENTRIES_PER_BUCKET)
-#define URL_HASH_BYTES \
- (BYTES_PER_ENTRY * (URL_HASH_ENTRIES + OVERFLOW_ENTRIES))
-#define CHANCE_OF_COLLISION \
- (((double)ENTRIES_PER_BUCKET)/((double)(1<<(8*3))))
-#define WANT_FILESIZE 0
-
-// NOTE: change to match BYTES_PER_ENTRY
-#define ENTRY_TAG(_x) \
- (((unsigned int)_x[0] << 16) + \
- ((unsigned int)_x[1] << 8) + \
- (unsigned int)_x[2])
-#define SET_ENTRY_TAG(_x,_t) \
- _x[0] = _t >> 16; \
- _x[1] = (_t >> 8) & 0xFF; \
- _x[2] = _t & 0xFF;
-
-#define MASK_TAG(_x) (_x & ((1U << (BYTES_PER_ENTRY * 8)))-1)
-
-#define BEGIN_HASH_LOOP \
- unsigned int bucket = (i % BUCKETS); \
- unsigned int tag = MASK_TAG((unsigned int)(i / BUCKETS)); \
- if (!tag) tag++; \
- unsigned char * base = bytes + bucket * BYTES_PER_BUCKET; \
- unsigned char * last = bytes + (bucket+1) * BYTES_PER_BUCKET - \
- BYTES_PER_ENTRY; \
- (void)last; \
- \
- for (unsigned int x = 0 ; x < ENTRIES_PER_BUCKET ; x++) { \
- unsigned char * e = base + x * BYTES_PER_ENTRY;
-
-#define BEGIN_OVERFLOW_HASH_LOOP \
- for (unsigned int j = 0 ; j < ENTRIES_PER_BUCKET ; j++) { \
- unsigned char * e = base + (URL_HASH_ENTRIES + j) * BYTES_PER_ENTRY;
-
-#define END_HASH_LOOP \
- }
-
-struct UrlHashTable {
-
- unsigned int numbytes;
- unsigned char * bytes;
- int fd;
-
- void zero() { memset(bytes, 0, numbytes); }
-
- void alloc(unsigned int want);
-
- void set(uint64_t i) {
- BEGIN_HASH_LOOP {
- if (!ENTRY_TAG(e)) {
- SET_ENTRY_TAG(e,tag);
- return;
- }
- } END_HASH_LOOP;
-
- fprintf(stderr, "url hash table overflow: %X, %X\n", (int)(base-bytes), tag);
-
- BEGIN_OVERFLOW_HASH_LOOP {
- if (!ENTRY_TAG(e)) {
- SET_ENTRY_TAG(e,tag);
- return;
- }
- } END_HASH_LOOP;
-
- ink_fatal(1, "overview entries overflow");
- }
-
- void clear(uint64_t i) {
- BEGIN_HASH_LOOP {
- if (ENTRY_TAG(e) == tag) {
- if (e != last)
- SET_ENTRY_TAG(e,ENTRY_TAG(last));
- SET_ENTRY_TAG(last,0);
- return;
- }
- } END_HASH_LOOP;
-
- fprintf(stderr, "url hash table entry to clear not found: %X, %X\n",
- (int)(base-bytes), tag);
- }
-
- int is_set(uint64_t i) {
- BEGIN_HASH_LOOP {
- if (ENTRY_TAG(e) == tag)
- return 1;
- } END_HASH_LOOP;
-
- if (ENTRY_TAG((last))) {
- BEGIN_OVERFLOW_HASH_LOOP {
- if (ENTRY_TAG(e) == tag)
- return 1;
- } END_HASH_LOOP;
- }
- return 0;
- }
-
- UrlHashTable();
-
- ~UrlHashTable();
-};
-UrlHashTable * uniq_urls = NULL;
-
-
-UrlHashTable::UrlHashTable() {
- unsigned long len = 0;
-
- if (!url_hash_entries)
- return;
-
- if (*url_hash_filename) {
- ink_assert((fd = open(url_hash_filename,O_RDWR|O_CREAT, 0644))>0);
- len = lseek(fd, 0, SEEK_END);
- }
-
- if (url_hash_entries > 0) {
- // if they specify the number of entries round it up
- url_hash_entries =
- (url_hash_entries + ENTRIES_PER_BUCKET - 1) & ~(ENTRIES_PER_BUCKET - 1);
- numbytes = URL_HASH_BYTES;
-
- // ensure it is either a new file or the correct size
- if (len != 0 && len != numbytes)
- panic("specified size != file size\n");
-
- } else {
-
- // otherwise make sure the file is non-zero and then use its
- // size as the size
- if (!len)
- panic("zero size URL Hash Table\n");
- if (len != URL_HASH_BYTES) {
- fprintf(stderr,
- "FATAL: hash file length (%lu) != URL_HASH_BYTES (%lu)\n",
- len, (unsigned long)URL_HASH_BYTES);
- exit(1);
- }
- numbytes = len;
- }
-
- if (*url_hash_filename) {
- ink_assert( !ftruncate(fd,numbytes) );
- bytes = (unsigned char *)
- mmap(NULL,numbytes,PROT_READ|PROT_WRITE,
- MAP_SHARED|MAP_NORESERVE,
- fd, 0);
- if (bytes == (unsigned char*)MAP_FAILED || !bytes)
- panic("unable to map URL Hash file\n");
- } else {
- bytes = (unsigned char *)malloc(numbytes);
- ink_assert(bytes);
- zero();
- }
-} // UrlHashTable::UrlHashTable
-
-
-UrlHashTable::~UrlHashTable()
-{
- ink_assert(!munmap((char*)bytes, numbytes));
- ink_assert(!close(fd));
-} // UrlHashTable::~UrlHashTable
-
-
-int seen_it(char * url) {
- if (!url_hash_entries)
- return 0;
- union {
- unsigned char md5[16];
- uint64_t i[2];
- } u;
- int l = 0;
- char * para = strrchr(url, '#');
- if (para)
- l = para - url;
- else
- l = strlen(url);
- ink_code_md5((unsigned char*)url,l,u.md5);
- uint64_t x = u.i[0] + u.i[1];
- if (uniq_urls->is_set(x)) {
- if (verbose) printf("YES: seen it '%s'\n", url);
- return 1;
- }
- uniq_urls->set(x);
- if (verbose) printf("NO: marked it '%s'\n", url);
- return 0;
-}
-
-static int make_url_client(char * url,char * base_url, bool seen,
- bool unthrottled)
-{
- int iport = 80;
- unsigned int ip = 0;
- char hostname[80],curl[512];
- char sche[8],host[512],port[10],path[512],frag[512],quer[512],para[512];
- int xsche,xhost,xport,xpath,xfrag,xquer,xpar,rel,slash;
-
- if (base_url) {
- ink_web_canonicalize_url(base_url, url, curl, 512);
- // hack for our own web server!
- if (curl[strlen(curl)-1] == 13)
- curl[strlen(curl)-1] = 0;
- if (curl[strlen(curl)-1] == 12)
- curl[strlen(curl)-1] = 0;
- } else
- strcpy(curl,url);
- if (!seen && seen_it(curl))
- return -1;
- ink_web_decompose_url(curl,sche,host,port,path,frag,quer,para,
- &xsche,&xhost,&xport,&xpath,&xfrag,&xquer,
- &xpar,&rel,&slash);
- if (follow_same) {
- if (!xhost || strcasecmp(host,current_host)) {
- if (verbose) printf("skipping %s\n",curl);
- return -1;
- }
- }
- if (!unthrottled && throttling_connections()) {
- defer_url(curl);
- return -1;
- }
- if (proxy_port) {
- iport = proxy_port;
- ip = proxy_addr;
- } else {
- if (xport) iport = atoi(port);
- if (!xhost) {
- if (verbose) fprintf(stderr, "bad url '%s'\n", curl);
- return -1;
- }
- ip = get_addr(host);
- if ((int)ip == -1) {
- if (verbose || verbose_errors)
- fprintf(stderr, "bad host '%s'\n", host);
- return -1;
- }
- }
- int sock = -1;
- if (keepalive)
- sock = get_ka(ip);
- if (sock < 0) {
- sock = make_client(ip, iport);
- fd[sock].keepalive = keepalive;
- } else {
- init_client(sock);
- current_clients++;
- fd[sock].keepalive--;
- }
- if (sock < 0)
- panic("cannot make client\n");
- char eheaders[16384];
- *eheaders = 0;
- int nheaders = extra_headers;
- memset(&eheaders,0,16384);
- if (nheaders>0) {
- char * eh = eheaders;
- if (!vary_user_agent) {
- eh += sprintf(eh, "User-Agent: Mozilla/4.04 [en] (X11; I; Linux 2.0.31 i586)\r\n");
- nheaders--;
- }
- if (nheaders>0)
- eh += sprintf(eh, "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, image/png, */*\r\n");
- while (--nheaders>0)
- eh += sprintf(eh, "Extra-Header%d: a lot of junk for header %d\r\n",
- nheaders, nheaders);
- }
- if (proxy_port)
- sprintf(fd[sock].req_header, "GET %s HTTP/1.0\r\n"
- "%s"
- "%s"
- "Accept: */*\r\n"
- "%s"
- "\r\n",
- curl,
- reload_rate > drand48() ? "Pragma: no-cache\r\n":"",
- fd[sock].keepalive?"Proxy-Connection: Keep-Alive\r\n":"",eheaders);
- else
- sprintf(fd[sock].req_header, "GET /%s%s%s%s%s HTTP/1.0\r\n"
- "Host: %s\r\n"
- "%s"
- "%s"
- "Accept: */*\r\n"
- "%s"
- "\r\n",
- path,xquer?"?":"",quer,xpar?";":"",para,host,
- reload_rate > drand48() ? "Pragma: no-cache\r\n":"",
- fd[sock].keepalive?"Connection: Keep-Alive\r\n":"",eheaders);
-
- if (verbose) printf("curl = '%s'\n",curl);
- if (show_before) printf("%s\n", curl);
- if (urlsdump_fp) fprintf(urlsdump_fp, "%s\n", curl);
- if (show_headers) printf("Request to Proxy: {\n%s}\n",fd[sock].req_header);
- char * epath = path + strlen(path);
- fd[sock].binary =
- !strncasecmp(path - 3, "gif", 3) || !strncasecmp(path - 3, "jpg", 3);
- fd[sock].response_length = 0;
- fd[sock].length = strlen(fd[sock].req_header);
- if (!fd[sock].response)
- fd[sock].response = (char*)malloc(MAX_BUFSIZE);
- strcpy(fd[sock].base_url, curl);
- return sock;
-}
-
-static FILE * get_defered_urls(FILE * fp) {
- char url[512];
- while (fgets(url,512,fp)) {
- if (n_defered_urls > MAX_DEFERED_URLS -2)
- return NULL;
- char * e = (char*)memchr(url,'\n', 512);
- if (e) *e = 0;
- make_url_client(url);
- }
- return fp;
-}
-
-int main(int argc, char *argv[]) {
- /* for QA -- we want to be able to tail an output file
- * during execution "nohup jtest -P pxy -p prt &"
- */
- setvbuf(stdout, (char*) NULL, _IOLBF, 0);
-
- fd = (FD*)malloc(MAXFDS * sizeof(FD));
- memset(fd,0,MAXFDS*sizeof(FD));
- process_args(argument_descriptions, n_argument_descriptions, argv);
- if (version) {
- show_version();
- exit(0);
- }
- if (!drand_seed)
- srand48((long)time(NULL));
- else
- srand48((long)drand_seed);
- if (zipf != 0.0)
- build_zipf();
- int max_fds = max_limit_fd();
- if (verbose) printf ("maximum of %d connections\n",max_fds);
- signal(SIGPIPE,SIG_IGN);
- start_time = now = ink_get_hrtime();
-
- urls_mode = n_file_arguments || *urls_file;
- nclients = client_rate? 0 : nclients;
-
- if (!local_host[0])
- ink_assert(!gethostname(local_host,80));
- local_addr = get_addr(local_host);
- if (!proxy_host[0])
- strcpy(proxy_host, local_host);
- if (proxy_port) {
- proxy_addr = get_addr(proxy_host);
- }
-
- if (!urls_mode) {
- if (compd_port) {
- build_response();
- open_server(compd_port, accept_compd);
- } else {
- if (!server_port)
- server_port = proxy_port + 1000;
- build_response();
- if (!only_clients) {
- for (int retry = 0 ; retry < 20 ; retry++) {
- server_fd = open_server(server_port + retry, accept_read);
- if (server_fd < 0) {
- if (server_fd == -EADDRINUSE) {
- printf("here\n");
- continue;
- }
- panic_perror("open_server");
- }
- break;
- }
- }
- bandwidth_test_to_go = bandwidth_test;
- if (!only_server) {
- if (proxy_port) {
- for (int i = 0 ; i < nclients ; i++)
- make_bfc_client(proxy_addr, proxy_port);
- }
- }
- }
- } else {
- if (check_content)
- build_response();
- follow = follow_arg;
- follow_same = follow_same_arg;
- uniq_urls = new UrlHashTable;
- defered_urls = (char**)malloc(sizeof(char*) * MAX_DEFERED_URLS);
- average_over = 1;
- if (*urlsdump_file) {
- urlsdump_fp = fopen(urlsdump_file,"w");
- if (!urlsdump_fp)
- panic_perror("fopen urlsdump file");
- }
- if (*urls_file) {
- FILE * fp = fopen(urls_file,"r");
- if (!fp)
- panic_perror("fopen urls file");
- if (get_defered_urls(fp))
- fclose(fp);
- else
- urls_fp = fp;
- }
- int i;
- for (i = 0; i < n_file_arguments; i++) {
- char sche[8],host[512],port[10],path[512],frag[512],quer[512],para[512];
- int xsche,xhost,xport,xpath,xfrag,xquer,xpar,rel,slash;
- ink_web_decompose_url(file_arguments[i],sche,host,port,
- path,frag,quer,para,
- &xsche,&xhost,&xport,&xpath,&xfrag,&xquer,
- &xpar,&rel,&slash);
- if (xhost) {
- strcpy(current_host,host);
- }
- }
- for (i = 0; i < n_file_arguments ; i++)
- make_url_client(file_arguments[i]);
- }
-
- int t = now / HRTIME_SECOND;
- int tclient = now / HRTIME_SECOND;
- int start = now / HRTIME_SECOND;
- while (1) {
- if (poll_loop()) break;
- int t2 = now / HRTIME_SECOND;
- if (urls_fp && n_defered_urls < MAX_DEFERED_URLS - DEFERED_URLS_BLOCK - 2){
- if (get_defered_urls(urls_fp)) {
- fclose(urls_fp);
- urls_fp = NULL;
- }
- }
- if ((!urls_mode || client_rate) && interval && t + interval <= t2) {
- t = t2;
- interval_report();
- }
- if (t2 != tclient) {
- for (int i = 0; i < client_rate * (t2 - tclient) ; i++)
- if (!urls_mode)
- make_bfc_client(proxy_addr, proxy_port);
- else
- undefer_url(true);
- tclient = t2;
- }
- if (test_time)
- if (t2 - start > test_time)
- done();
- if (is_done())
- done();
- }
-
- return 0;
-}
-
-
-/*---------------------------------------------------------------------------*
-
- int ink_web_decompose_url(...)
-
- This function takes an input URL in src_url and splits it into its
- component parts, including a scheme, host, port, path, fragment,
- query, and parameters. you must pass in buffers for each of these.
- If you pass in a NULL pointer for any of these, it will not be
- return
<TRUNCATED>