You are viewing a plain text version of this content. The canonical link for it is here.
Posted to cvs@httpd.apache.org by ic...@apache.org on 2021/10/28 12:50:03 UTC
svn commit: r1894599 [2/2] - in /httpd/httpd/trunk/test: ./ modules/
modules/core/ modules/http2/ pyhttpd/ pyhttpd/conf/ pyhttpd/htdocs/ssl/
Modified: httpd/httpd/trunk/test/pyhttpd/env.py
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/test/pyhttpd/env.py?rev=1894599&r1=1894598&r2=1894599&view=diff
==============================================================================
--- httpd/httpd/trunk/test/pyhttpd/env.py (original)
+++ httpd/httpd/trunk/test/pyhttpd/env.py Thu Oct 28 12:50:02 2021
@@ -9,14 +9,13 @@ import sys
import time
from datetime import datetime, timedelta
from string import Template
-from typing import List
-
-import requests
+from typing import List, Optional
from configparser import ConfigParser, ExtendedInterpolation
from urllib.parse import urlparse
from .certs import Credentials, HttpdTestCA, CertificateSpec
+from .log import HttpdErrorLog
from .nghttp import Nghttp
from .result import ExecResult
@@ -56,7 +55,6 @@ class HttpdTestSetup:
"headers",
"setenvif",
"slotmem_shm",
- "ssl",
"status",
"autoindex",
"cgid",
@@ -79,8 +77,13 @@ class HttpdTestSetup:
mod_names = modules.copy() if modules else self.MODULES.copy()
if add_modules:
mod_names.extend(add_modules)
+ if self.env.mpm_module is not None and self.env.mpm_module not in mod_names:
+ mod_names.append(self.env.mpm_module)
+ if self.env.ssl_module is not None and self.env.ssl_module not in mod_names:
+ mod_names.append(self.env.ssl_module)
self._make_modules_conf(modules=mod_names)
self._make_htdocs()
+ self.env.clear_curl_headerfiles()
def _make_dirs(self):
if os.path.exists(self.env.gen_dir):
@@ -116,10 +119,18 @@ class HttpdTestSetup:
modules_conf = os.path.join(self.env.server_dir, 'conf/modules.conf')
with open(modules_conf, 'w') as fd:
# issue load directives for all modules we want that are shared
+ missing_mods = list()
for m in modules:
mod_path = os.path.join(self.env.libexec_dir, f"mod_{m}.so")
if os.path.isfile(mod_path):
fd.write(f"LoadModule {m}_module \"{mod_path}\"\n")
+ elif m in self.env.static_modules:
+ fd.write(f"#built static: LoadModule {m}_module \"{mod_path}\"\n")
+ else:
+ missing_mods.append(m)
+ if len(missing_mods) > 0:
+ raise Exception(f"Unable to find modules: {missing_mods} "
+ f"DSOs: {self.env.dso_modules}")
def _make_htdocs(self):
our_dir = os.path.dirname(inspect.getfile(Dummy))
@@ -138,14 +149,19 @@ class HttpdTestSetup:
class HttpdTestEnv:
+ @classmethod
+ def get_ssl_module(cls):
+ return os.environ['SSL'] if 'SSL' in os.environ else 'ssl'
+
def __init__(self, pytestconfig=None,
- local_dir=None, add_base_conf: str = None,
+ local_dir=None, add_base_conf: List[str] = None,
interesting_modules: List[str] = None):
self._our_dir = os.path.dirname(inspect.getfile(Dummy))
self._local_dir = local_dir if local_dir else self._our_dir
self.config = ConfigParser(interpolation=ExtendedInterpolation())
self.config.read(os.path.join(self._our_dir, 'config.ini'))
+ self._bin_dir = self.config.get('global', 'bindir')
self._apxs = self.config.get('global', 'apxs')
self._prefix = self.config.get('global', 'prefix')
self._apachectl = self.config.get('global', 'apachectl')
@@ -157,6 +173,7 @@ class HttpdTestEnv:
self._http_port = int(self.config.get('test', 'http_port'))
self._https_port = int(self.config.get('test', 'https_port'))
+ self._proxy_port = int(self.config.get('test', 'proxy_port'))
self._http_tld = self.config.get('test', 'http_tld')
self._test_dir = self.config.get('test', 'test_dir')
self._gen_dir = self.config.get('test', 'gen_dir')
@@ -165,40 +182,38 @@ class HttpdTestEnv:
self._server_docs_dir = os.path.join(self._server_dir, "htdocs")
self._server_logs_dir = os.path.join(self.server_dir, "logs")
self._server_access_log = os.path.join(self._server_logs_dir, "access_log")
- self._server_error_log = os.path.join(self._server_logs_dir, "error_log")
+ self._error_log = HttpdErrorLog(os.path.join(self._server_logs_dir, "error_log"))
+ self._apachectl_stderr = None
- self._dso_modules = self.config.get('global', 'dso_modules').split(' ')
- self._mpm_type = os.environ['MPM'] if 'MPM' in os.environ else 'event'
+ self._dso_modules = self.config.get('httpd', 'dso_modules').split(' ')
+ self._static_modules = self.config.get('httpd', 'static_modules').split(' ')
+ self._mpm_module = f"mpm_{os.environ['MPM']}" if 'MPM' in os.environ else 'mpm_event'
+ self._ssl_module = self.get_ssl_module()
+ if len(self._ssl_module.strip()) == 0:
+ self._ssl_module = None
self._httpd_addr = "127.0.0.1"
self._http_base = f"http://{self._httpd_addr}:{self.http_port}"
self._https_base = f"https://{self._httpd_addr}:{self.https_port}"
self._test_conf = os.path.join(self._server_conf_dir, "test.conf")
- self._httpd_base_conf = f"""
- LoadModule mpm_{self.mpm_type}_module \"{self.libexec_dir}/mod_mpm_{self.mpm_type}.so\"
- <IfModule mod_ssl.c>
- SSLSessionCache "shmcb:ssl_gcache_data(32000)"
- </IfModule>
- """
+ self._httpd_base_conf = []
if add_base_conf:
- self._httpd_base_conf += f"\n{add_base_conf}"
+ self._httpd_base_conf.extend(add_base_conf)
self._verbosity = pytestconfig.option.verbose if pytestconfig is not None else 0
if self._verbosity >= 2:
log_level = "trace2"
- self._httpd_base_conf += f"""
- LogLevel core:trace5 mpm_{self.mpm_type}:trace5
- """
+ self._httpd_base_conf .append(f"LogLevel core:trace5 {self.mpm_module}:trace5")
elif self._verbosity >= 1:
log_level = "debug"
else:
log_level = "info"
if interesting_modules:
- self._httpd_base_conf += "\nLogLevel"
+ l = "LogLevel"
for name in interesting_modules:
- self._httpd_base_conf += f" {name}:{log_level}"
- self._httpd_base_conf += "\n"
+ l += f" {name}:{log_level}"
+ self._httpd_base_conf.append(l)
self._ca = None
self._cert_specs = [CertificateSpec(domains=[
@@ -209,6 +224,7 @@ class HttpdTestEnv:
], key_type='rsa4096')]
self._verify_certs = False
+ self._curl_headerfiles_n = 0
@property
def apxs(self) -> str:
@@ -223,8 +239,16 @@ class HttpdTestEnv:
return self._prefix
@property
- def mpm_type(self) -> str:
- return self._mpm_type
+ def mpm_module(self) -> str:
+ return self._mpm_module
+
+ @property
+ def ssl_module(self) -> str:
+ return self._ssl_module
+
+ @property
+ def http_addr(self) -> str:
+ return self._httpd_addr
@property
def http_port(self) -> int:
@@ -235,6 +259,10 @@ class HttpdTestEnv:
return self._https_port
@property
+ def proxy_port(self) -> int:
+ return self._proxy_port
+
+ @property
def http_tld(self) -> str:
return self._http_tld
@@ -247,6 +275,10 @@ class HttpdTestEnv:
return self._https_base
@property
+ def bin_dir(self) -> str:
+ return self._bin_dir
+
+ @property
def gen_dir(self) -> str:
return self._gen_dir
@@ -275,6 +307,10 @@ class HttpdTestEnv:
return self._dso_modules
@property
+ def static_modules(self) -> List[str]:
+ return self._static_modules
+
+ @property
def server_conf_dir(self) -> str:
return self._server_conf_dir
@@ -283,9 +319,13 @@ class HttpdTestEnv:
return self._server_docs_dir
@property
- def httpd_base_conf(self) -> str:
+ def httpd_base_conf(self) -> List[str]:
return self._httpd_base_conf
+ @property
+ def httpd_error_log(self) -> HttpdErrorLog:
+ return self._error_log
+
def local_src(self, path):
return os.path.join(self.local_dir, path)
@@ -300,13 +340,18 @@ class HttpdTestEnv:
def ca(self) -> Credentials:
return self._ca
+ @property
+ def apachectl_stderr(self):
+ return self._apachectl_stderr
+
def add_cert_specs(self, specs: List[CertificateSpec]):
self._cert_specs.extend(specs)
def issue_certs(self):
if self._ca is None:
self._ca = HttpdTestCA.create_root(name=self.http_tld,
- store_dir=os.path.join(self.server_dir, 'ca'), key_type="rsa4096")
+ store_dir=os.path.join(self.server_dir, 'ca'),
+ key_type="rsa4096")
self._ca.issue_certs(self._cert_specs)
def get_credentials_for_name(self, dns_name) -> List['Credentials']:
@@ -315,6 +360,13 @@ class HttpdTestEnv:
return self.ca.get_credentials_for_name(spec.domains[0])
return []
+ def _versiontuple(self, v):
+ return tuple(map(int, v.split('.')))
+
+ def httpd_is_at_least(self, minv):
+ hv = self._versiontuple(self.get_httpd_version())
+ return hv >= self._versiontuple(minv)
+
def has_h2load(self):
return self._h2load != ""
@@ -344,158 +396,170 @@ class HttpdTestEnv:
if not os.path.exists(path):
return os.makedirs(path)
- def run(self, args) -> ExecResult:
- log.debug("execute: %s", " ".join(args))
+ def run(self, args, input=None, debug_log=True):
+ if debug_log:
+ log.debug(f"run: {args}")
start = datetime.now()
- p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
- return ExecResult(exit_code=p.returncode, stdout=p.stdout, stderr=p.stderr,
+ p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE,
+ input=input.encode() if input else None)
+ return ExecResult(args=args, exit_code=p.returncode,
+ stdout=p.stdout, stderr=p.stderr,
duration=datetime.now() - start)
def mkurl(self, scheme, hostname, path='/'):
port = self.https_port if scheme == 'https' else self.http_port
- return "%s://%s.%s:%s%s" % (scheme, hostname, self.http_tld, port, path)
+ return f"{scheme}://{hostname}.{self.http_tld}:{port}{path}"
- def install_test_conf(self, conf: List[str]):
+ def install_test_conf(self, lines: List[str]):
with open(self._test_conf, 'w') as fd:
- fd.write(f"{self.httpd_base_conf}\n")
- for line in conf:
- fd.write(f"{line}\n")
-
- def is_live(self, url, timeout: timedelta = None):
- s = requests.Session()
- if not timeout:
- timeout = timedelta(seconds=10)
+ fd.write('\n'.join(self._httpd_base_conf))
+ fd.write('\n')
+ fd.write('\n'.join(lines))
+ fd.write('\n')
+
+ def is_live(self, url: str = None, timeout: timedelta = None):
+ if url is None:
+ url = self._http_base
+ if timeout is None:
+ timeout = timedelta(seconds=5)
try_until = datetime.now() + timeout
- log.debug("checking reachability of %s", url)
+ last_err = ""
while datetime.now() < try_until:
+ # noinspection PyBroadException
try:
- req = requests.Request('HEAD', url).prepare()
- s.send(req, verify=self._verify_certs, timeout=timeout.total_seconds())
- return True
- except IOError:
- log.debug("connect error: %s", sys.exc_info()[0])
- time.sleep(.2)
+ r = self.curl_get(url, insecure=True, debug_log=False)
+ if r.exit_code == 0:
+ return True
+ time.sleep(.1)
+ except ConnectionRefusedError:
+ log.debug("connection refused")
+ time.sleep(.1)
except:
- log.warning("Unexpected error: %s", sys.exc_info()[0])
- time.sleep(.2)
- log.debug(f"Unable to contact '{url}' after {timeout} sec")
+ if last_err != str(sys.exc_info()[0]):
+ last_err = str(sys.exc_info()[0])
+ log.debug("Unexpected error: %s", last_err)
+ time.sleep(.1)
+ log.debug(f"Unable to contact server after {timeout}")
return False
- def is_dead(self, url, timeout: timedelta = None):
- s = requests.Session()
- if not timeout:
- timeout = timedelta(seconds=10)
+ def is_dead(self, url: str = None, timeout: timedelta = None):
+ if url is None:
+ url = self._http_base
+ if timeout is None:
+ timeout = timedelta(seconds=5)
try_until = datetime.now() + timeout
- log.debug("checking reachability of %s", url)
+ last_err = None
while datetime.now() < try_until:
+ # noinspection PyBroadException
try:
- req = requests.Request('HEAD', url).prepare()
- s.send(req, verify=self._verify_certs, timeout=int(timeout.total_seconds()))
- time.sleep(.2)
- except IOError:
+ r = self.curl_get(url, debug_log=False)
+ if r.exit_code != 0:
+ return True
+ time.sleep(.1)
+ except ConnectionRefusedError:
+ log.debug("connection refused")
return True
- log.debug("Server still responding after %d sec", timeout)
+ except:
+ if last_err != str(sys.exc_info()[0]):
+ last_err = str(sys.exc_info()[0])
+ log.debug("Unexpected error: %s", last_err)
+ time.sleep(.1)
+ log.debug(f"Server still responding after {timeout}")
return False
- def _run_apachectl(self, cmd):
+ def _run_apachectl(self, cmd) -> ExecResult:
args = [self._apachectl,
"-d", self.server_dir,
"-f", os.path.join(self._server_dir, 'conf/httpd.conf'),
"-k", cmd]
- log.debug("execute: %s", " ".join(args))
- p = subprocess.run(args, capture_output=True, text=True)
- rv = p.returncode
- if rv != 0:
- log.warning(f"exit {rv}, stdout: {p.stdout}, stderr: {p.stderr}")
- return rv
+ r = self.run(args)
+ self._apachectl_stderr = r.stderr
+ if r.exit_code != 0:
+ log.warning(f"failed: {r}")
+ return r
def apache_reload(self):
- rv = self._run_apachectl("graceful")
- if rv == 0:
+ r = self._run_apachectl("graceful")
+ if r.exit_code == 0:
timeout = timedelta(seconds=10)
- rv = 0 if self.is_live(self._http_base, timeout=timeout) else -1
- return rv
+ return 0 if self.is_live(self._http_base, timeout=timeout) else -1
+ return r.exit_code
def apache_restart(self):
self.apache_stop()
- rv = self._run_apachectl("start")
- if rv == 0:
+ r = self._run_apachectl("start")
+ if r.exit_code == 0:
timeout = timedelta(seconds=10)
- rv = 0 if self.is_live(self._http_base, timeout=timeout) else -1
- return rv
+ return 0 if self.is_live(self._http_base, timeout=timeout) else -1
+ return r.exit_code
def apache_stop(self):
- rv = self._run_apachectl("stop")
- if rv == 0:
+ r = self._run_apachectl("stop")
+ if r.exit_code == 0:
timeout = timedelta(seconds=10)
- rv = 0 if self.is_dead(self._http_base, timeout=timeout) else -1
- log.debug("waited for a apache.is_dead, rv=%d", rv)
+ return 0 if self.is_dead(self._http_base, timeout=timeout) else -1
+ return r
+
+ def apache_graceful_stop(self):
+ log.debug("stop apache")
+ self._run_apachectl("graceful-stop")
+ return 0 if self.is_dead() else -1
+
+ def apache_fail(self):
+ log.debug("expect apache fail")
+ self._run_apachectl("stop")
+ rv = self._run_apachectl("start")
+ if rv == 0:
+ rv = 0 if self.is_dead() else -1
+ else:
+ rv = 0
return rv
def apache_access_log_clear(self):
if os.path.isfile(self._server_access_log):
os.remove(self._server_access_log)
- def apache_error_log_clear(self):
- if os.path.isfile(self._server_error_log):
- os.remove(self._server_error_log)
-
- RE_APLOGNO = re.compile(r'.*\[(?P<module>[^:]+):(error|warn)].* (?P<aplogno>AH\d+): .+')
- RE_SSL_LIB_ERR = re.compile(r'.*\[ssl:error].* SSL Library Error: error:(?P<errno>\S+):.+')
- RE_ERRLOG_ERROR = re.compile(r'.*\[(?P<module>[^:]+):error].*')
- RE_ERRLOG_WARN = re.compile(r'.*\[(?P<module>[^:]+):warn].*')
-
- def apache_errors_and_warnings(self):
- errors = []
- warnings = []
-
- if os.path.isfile(self._server_error_log):
- for line in open(self._server_error_log):
- m = self.RE_APLOGNO.match(line)
- if m and m.group('aplogno') in [
- 'AH02032',
- 'AH01276',
- 'AH01630',
- 'AH00135',
- 'AH02261', # Re-negotiation handshake failed (our test_101
- ]:
- # we know these happen normally in our tests
- continue
- m = self.RE_SSL_LIB_ERR.match(line)
- if m and m.group('errno') in [
- '1417A0C1', # cipher suite mismatch, test_101
- '1417C0C7', # client cert not accepted, test_101
- ]:
- # we know these happen normally in our tests
- continue
- m = self.RE_ERRLOG_ERROR.match(line)
- if m and m.group('module') not in ['cgid']:
- errors.append(line)
- continue
- m = self.RE_ERRLOG_WARN.match(line)
- if m:
- warnings.append(line)
- continue
- return errors, warnings
+ def get_ca_pem_file(self, hostname: str) -> Optional[str]:
+ if len(self.get_credentials_for_name(hostname)) > 0:
+ return self.ca.cert_file
+ return None
+
+ def clear_curl_headerfiles(self):
+ for fname in os.listdir(path=self.gen_dir):
+ if re.match(r'curl\.headers\.\d+', fname):
+ os.remove(os.path.join(self.gen_dir, fname))
+ self._curl_headerfiles_n = 0
- def curl_complete_args(self, urls, timeout, options):
+ def curl_complete_args(self, urls, timeout=None, options=None,
+ insecure=False, force_resolve=True):
if not isinstance(urls, list):
urls = [urls]
u = urlparse(urls[0])
assert u.hostname, f"hostname not in url: {urls[0]}"
- assert u.port, f"port not in url: {urls[0]}"
- headerfile = ("%s/curl.headers" % self.gen_dir)
- if os.path.isfile(headerfile):
- os.remove(headerfile)
-
- args = [
- self._curl,
- "--cacert", self.ca.cert_file,
- "-s", "-D", headerfile,
- "--resolve", ("%s:%s:%s" % (u.hostname, u.port, self._httpd_addr)),
- "--connect-timeout", ("%d" % timeout),
- "--path-as-is"
+ headerfile = f"{self.gen_dir}/curl.headers.{self._curl_headerfiles_n}"
+ self._curl_headerfiles_n += 1
+
+ args = [
+ self._curl, "-s", "--path-as-is", "-D", headerfile,
]
+ if u.scheme == 'http':
+ pass
+ elif insecure:
+ args.append('--insecure')
+ elif options and "--cacert" in options:
+ pass
+ else:
+ ca_pem = self.get_ca_pem_file(u.hostname)
+ if ca_pem:
+ args.extend(["--cacert", ca_pem])
+
+ if force_resolve and u.hostname != 'localhost' \
+ and u.hostname != self._httpd_addr \
+ and not re.match(r'^(\d+|\[|:).*', u.hostname):
+ assert u.port, f"port not in url: {urls[0]}"
+ args.extend(["--resolve", f"{u.hostname}:{u.port}:{self._httpd_addr}"])
+ if timeout is not None and int(timeout) > 0:
+ args.extend(["--connect-timeout", str(int(timeout))])
if options:
args.extend(options)
args += urls
@@ -505,7 +569,7 @@ class HttpdTestEnv:
lines = open(headerfile).readlines()
exp_stat = True
if r is None:
- r = ExecResult(exit_code=0, stdout=b'', stderr=b'')
+ r = ExecResult(args=[], exit_code=0, stdout=b'', stderr=b'')
header = {}
for line in lines:
if exp_stat:
@@ -527,23 +591,29 @@ class HttpdTestEnv:
m = re.match(r'^([^:]+):\s*(.*)$', line)
assert m
header[m.group(1).lower()] = m.group(2)
- r.response["header"] = header
+ if r.response:
+ r.response["header"] = header
return r
- def curl_raw(self, urls, timeout, options):
+ def curl_raw(self, urls, timeout=10, options=None, insecure=False,
+ debug_log=True, force_resolve=True):
xopt = ['-vvvv']
if options:
xopt.extend(options)
- args, headerfile = self.curl_complete_args(urls, timeout, xopt)
+ args, headerfile = self.curl_complete_args(
+ urls=urls, timeout=timeout, options=options, insecure=insecure,
+ force_resolve=force_resolve)
r = self.run(args)
if r.exit_code == 0:
self.curl_parse_headerfile(headerfile, r=r)
if r.json:
r.response["json"] = r.json
+ os.remove(headerfile)
return r
- def curl_get(self, url, timeout=5, options=None):
- return self.curl_raw([url], timeout=timeout, options=options)
+ def curl_get(self, url, insecure=False, debug_log=True, options=None):
+ return self.curl_raw([url], insecure=insecure,
+ options=options, debug_log=debug_log)
def curl_upload(self, url, fpath, timeout=5, options=None):
if not options:
@@ -551,7 +621,7 @@ class HttpdTestEnv:
options.extend([
"--form", ("file=@%s" % fpath)
])
- return self.curl_raw([url], timeout, options)
+ return self.curl_raw(urls=[url], timeout=timeout, options=options)
def curl_post_data(self, url, data="", timeout=5, options=None):
if not options:
Added: httpd/httpd/trunk/test/pyhttpd/log.py
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/test/pyhttpd/log.py?rev=1894599&view=auto
==============================================================================
--- httpd/httpd/trunk/test/pyhttpd/log.py (added)
+++ httpd/httpd/trunk/test/pyhttpd/log.py Thu Oct 28 12:50:02 2021
@@ -0,0 +1,163 @@
+import os
+import re
+import time
+from datetime import datetime, timedelta
+from io import SEEK_END
+from typing import List, Tuple, Any
+
+
+class HttpdErrorLog:
+ """Checking the httpd error log for errors and warnings, including
+ limiting checks from a last known position forward.
+ """
+
+ RE_ERRLOG_ERROR = re.compile(r'.*\[(?P<module>[^:]+):error].*')
+ RE_ERRLOG_WARN = re.compile(r'.*\[(?P<module>[^:]+):warn].*')
+ RE_APLOGNO = re.compile(r'.*\[(?P<module>[^:]+):(error|warn)].* (?P<aplogno>AH\d+): .+')
+ RE_SSL_LIB_ERR = re.compile(r'.*\[ssl:error].* SSL Library Error: error:(?P<errno>\S+):.+')
+
+ def __init__(self, path: str):
+ self._path = path
+ self._ignored_modules = []
+ self._ignored_lognos = set()
+ self._ignored_patterns = []
+ # remember the file position we started with
+ self._start_pos = 0
+ if os.path.isfile(self._path):
+ with open(self._path) as fd:
+ self._start_pos = fd.seek(0, SEEK_END)
+ self._last_pos = self._start_pos
+ self._last_errors = []
+ self._last_warnings = []
+ self._observed_erros = set()
+ self._observed_warnings = set()
+
+ def __repr__(self):
+ return f"HttpdErrorLog[{self._path}, errors: {' '.join(self._last_errors)}, " \
+ f"warnings: {' '.join(self._last_warnings)}]"
+
+ @property
+ def path(self) -> str:
+ return self._path
+
+ def clear_log(self):
+ if os.path.isfile(self.path):
+ os.remove(self.path)
+ self._start_pos = 0
+ self._last_pos = self._start_pos
+ self._last_errors = []
+ self._last_warnings = []
+ self._observed_erros = set()
+ self._observed_warnings = set()
+
+ def set_ignored_modules(self, modules: List[str]):
+ self._ignored_modules = modules.copy() if modules else []
+
+ def set_ignored_lognos(self, lognos: List[str]):
+ if lognos:
+ for l in lognos:
+ self._ignored_lognos.add(l)
+
+ def add_ignored_patterns(self, patterns: List[Any]):
+ self._ignored_patterns.extend(patterns)
+
+ def _is_ignored(self, line: str) -> bool:
+ for p in self._ignored_patterns:
+ if p.match(line):
+ return True
+ m = self.RE_APLOGNO.match(line)
+ if m and m.group('aplogno') in self._ignored_lognos:
+ return True
+ return False
+
+ def get_recent(self, advance=True) -> Tuple[List[str], List[str]]:
+ """Collect error and warning from the log since the last remembered position
+ :param advance: advance the position to the end of the log afterwards
+ :return: list of error and list of warnings as tuple
+ """
+ self._last_errors = []
+ self._last_warnings = []
+ if os.path.isfile(self._path):
+ with open(self._path) as fd:
+ fd.seek(self._last_pos, os.SEEK_SET)
+ for line in fd:
+ if self._is_ignored(line):
+ continue
+ m = self.RE_ERRLOG_ERROR.match(line)
+ if m and m.group('module') not in self._ignored_modules:
+ self._last_errors.append(line)
+ continue
+ m = self.RE_ERRLOG_WARN.match(line)
+ if m:
+ if m and m.group('module') not in self._ignored_modules:
+ self._last_warnings.append(line)
+ continue
+ if advance:
+ self._last_pos = fd.tell()
+ self._observed_erros.update(set(self._last_errors))
+ self._observed_warnings.update(set(self._last_warnings))
+ return self._last_errors, self._last_warnings
+
+ def get_recent_count(self, advance=True):
+ errors, warnings = self.get_recent(advance=advance)
+ return len(errors), len(warnings)
+
+ def ignore_recent(self):
+ """After a test case triggered errors/warnings on purpose, add
+ those to our 'observed' list so the do not get reported as 'missed'.
+ """
+ self._last_errors = []
+ self._last_warnings = []
+ if os.path.isfile(self._path):
+ with open(self._path) as fd:
+ fd.seek(self._last_pos, os.SEEK_SET)
+ for line in fd:
+ if self._is_ignored(line):
+ continue
+ m = self.RE_ERRLOG_ERROR.match(line)
+ if m and m.group('module') not in self._ignored_modules:
+ self._observed_erros.add(line)
+ continue
+ m = self.RE_ERRLOG_WARN.match(line)
+ if m:
+ if m and m.group('module') not in self._ignored_modules:
+ self._observed_warnings.add(line)
+ continue
+ self._last_pos = fd.tell()
+
+ def get_missed(self) -> Tuple[List[str], List[str]]:
+ errors = []
+ warnings = []
+ if os.path.isfile(self._path):
+ with open(self._path) as fd:
+ fd.seek(self._start_pos, os.SEEK_SET)
+ for line in fd:
+ if self._is_ignored(line):
+ continue
+ m = self.RE_ERRLOG_ERROR.match(line)
+ if m and m.group('module') not in self._ignored_modules \
+ and line not in self._observed_erros:
+ errors.append(line)
+ continue
+ m = self.RE_ERRLOG_WARN.match(line)
+ if m:
+ if m and m.group('module') not in self._ignored_modules \
+ and line not in self._observed_warnings:
+ warnings.append(line)
+ continue
+ return errors, warnings
+
+ def scan_recent(self, pattern: re, timeout=10):
+ if not os.path.isfile(self.path):
+ return False
+ with open(self.path) as fd:
+ end = datetime.now() + timedelta(seconds=timeout)
+ while True:
+ fd.seek(self._last_pos, os.SEEK_SET)
+ for line in fd:
+ if pattern.match(line):
+ return True
+ if datetime.now() > end:
+ raise TimeoutError(f"pattern not found in error log after {timeout} seconds")
+ time.sleep(.1)
+ return False
Modified: httpd/httpd/trunk/test/pyhttpd/nghttp.py
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/test/pyhttpd/nghttp.py?rev=1894599&r1=1894598&r2=1894599&view=diff
==============================================================================
--- httpd/httpd/trunk/test/pyhttpd/nghttp.py (original)
+++ httpd/httpd/trunk/test/pyhttpd/nghttp.py Thu Oct 28 12:50:02 2021
@@ -84,12 +84,12 @@ class Nghttp:
if len(l) == 0:
body += '\n'
continue
- m = re.match(r'\[(.*)] recv \(stream_id=(\d+)\) (\S+): (\S*)', l)
+ m = re.match(r'\[.*] recv \(stream_id=(\d+)\) (\S+): (\S*)', l)
if m:
- s = self.get_stream(streams, m.group(2))
- hname = m.group(3)
- hval = m.group(4)
- print(f"{m.group(1)}: stream {s['id']} header {hname}: {hval}")
+ s = self.get_stream(streams, m.group(1))
+ hname = m.group(2)
+ hval = m.group(3)
+ print("stream %d header %s: %s" % (s["id"], hname, hval))
header = s["header"]
if hname in header:
header[hname] += ", %s" % hval
@@ -98,11 +98,11 @@ class Nghttp:
body = ''
continue
- m = re.match(r'\[(.*)] recv HEADERS frame <.* stream_id=(\d+)>', l)
+ m = re.match(r'\[.*] recv HEADERS frame <.* stream_id=(\d+)>', l)
if m:
- s = self.get_stream(streams, m.group(2))
+ s = self.get_stream(streams, m.group(1))
if s:
- print(f"{m.group(1)}: recv HEADERS on stream {s['id']} with {len(s['header'])} fields")
+ print("stream %d: recv %d header" % (s["id"], len(s["header"])))
response = s["response"]
hkey = "header"
if "header" in response:
@@ -121,13 +121,13 @@ class Nghttp:
body = ''
continue
- m = re.match(r'(.*)\[(.*)] recv DATA frame <length=(\d+), .*stream_id=(\d+)>', l)
+ m = re.match(r'(.*)\[.*] recv DATA frame <length=(\d+), .*stream_id=(\d+)>', l)
if m:
- s = self.get_stream(streams, m.group(4))
+ s = self.get_stream(streams, m.group(3))
body += m.group(1)
- blen = int(m.group(3))
+ blen = int(m.group(2))
if s:
- print(f"{m.group(2)}: recv DATA on stream {s['id']} with {blen} bytes")
+ print("stream %d: %d DATA bytes added" % (s["id"], blen))
padlen = 0
if len(lines) > lidx + 2:
mpad = re.match(r' +\(padlen=(\d+)\)', lines[lidx+2])
@@ -140,14 +140,14 @@ class Nghttp:
skip_indents = True
continue
- m = re.match(r'\[(.*)] recv PUSH_PROMISE frame <.* stream_id=(\d+)>', l)
+ m = re.match(r'\[.*] recv PUSH_PROMISE frame <.* stream_id=(\d+)>', l)
if m:
- s = self.get_stream(streams, m.group(2))
+ s = self.get_stream(streams, m.group(1))
if s:
# headers we have are request headers for the PUSHed stream
# these have been received on the originating stream, the promised
# stream id it mentioned in the following lines
- print(f"{m.group(1)}: recv PUSH_PROMISE on stream {s['id']} with {len(s['header'])} header")
+ print("stream %d: %d PUSH_PROMISE header" % (s["id"], len(s["header"])))
if len(lines) > lidx+2:
m2 = re.match(r'\s+\(.*promised_stream_id=(\d+)\)', lines[lidx+2])
if m2:
@@ -157,16 +157,16 @@ class Nghttp:
s["header"] = {}
continue
- m = re.match(r'(.*)\[(.*)] recv (\S+) frame <length=(\d+), .*stream_id=(\d+)>', l)
+ m = re.match(r'(.*)\[.*] recv (\S+) frame <length=(\d+), .*stream_id=(\d+)>', l)
if m:
- print(f"{m.group(2)}: recv frame {m.group(3)} on stream {m.group(5)}")
+ print("recv frame %s on stream %s" % (m.group(2), m.group(4)))
body += m.group(1)
skip_indents = True
continue
- m = re.match(r'(.*)\[(.*)] send (\S+) frame <length=(\d+), .*stream_id=(\d+)>', l)
+ m = re.match(r'(.*)\[.*] send (\S+) frame <length=(\d+), .*stream_id=(\d+)>', l)
if m:
- print(f"{m.group(2)}: send frame {m.group(3)} on stream {m.group(5)}")
+ print("send frame %s on stream %s" % (m.group(2), m.group(4)))
body += m.group(1)
skip_indents = True
continue
@@ -284,5 +284,6 @@ Content-Transfer-Encoding: binary
print(("execute: %s" % " ".join(args)))
start = datetime.now()
p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
- return ExecResult(exit_code=p.returncode, stdout=p.stdout, stderr=p.stderr,
+ return ExecResult(args=args, exit_code=p.returncode,
+ stdout=p.stdout, stderr=p.stderr,
duration=datetime.now() - start)
Modified: httpd/httpd/trunk/test/pyhttpd/result.py
URL: http://svn.apache.org/viewvc/httpd/httpd/trunk/test/pyhttpd/result.py?rev=1894599&r1=1894598&r2=1894599&view=diff
==============================================================================
--- httpd/httpd/trunk/test/pyhttpd/result.py (original)
+++ httpd/httpd/trunk/test/pyhttpd/result.py Thu Oct 28 12:50:02 2021
@@ -5,7 +5,9 @@ from typing import Optional, Dict, List
class ExecResult:
- def __init__(self, exit_code: int, stdout: bytes, stderr: bytes = None, duration: timedelta = None):
+ def __init__(self, args: List[str], exit_code: int,
+ stdout: bytes, stderr: bytes = None, duration: timedelta = None):
+ self._args = args
self._exit_code = exit_code
self._raw = stdout if stdout else b''
self._stdout = stdout.decode() if stdout is not None else ""
@@ -20,11 +22,18 @@ class ExecResult:
except:
self._json_out = None
+ def __repr__(self):
+ return f"ExecResult[code={self.exit_code}, args={self._args}, stdout={self.stdout}, stderr={self.stderr}]"
+
@property
def exit_code(self) -> int:
return self._exit_code
@property
+ def args(self) -> List[str]:
+ return self._args
+
+ @property
def outraw(self) -> bytes:
return self._raw