You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@qpid.apache.org by ch...@apache.org on 2020/09/24 14:38:43 UTC
[qpid-dispatch] branch master updated: DISPATCH-1781: Scraper
injects any log module output into data web page
This is an automated email from the ASF dual-hosted git repository.
chug pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/qpid-dispatch.git
The following commit(s) were added to refs/heads/master by this push:
new 0207301 DISPATCH-1781: Scraper injects any log module output into data web page
0207301 is described below
commit 0207301f8d092fb661862c452a1faef90b794cc6
Author: Chuck Rolke <ch...@apache.org>
AuthorDate: Thu Sep 24 09:43:18 2020 -0400
DISPATCH-1781: Scraper injects any log module output into data web page
Add command line settings to specify a list of log modules instead of hard-coding 'SCRAPER' as the injected log module. These log module statements are injected verbatim into the main scraper web page.
Command line adds:
--log-modules LOG_MODULES, -lm LOG_MODULES
Include list of named modules log entries in main log
display. Specify multiple modules as a CSV string.
Defaults to "SCRAPER"
This will be useful for upcoming protocol adaptors.
scraper --log-modules TCP_ADAPTOR -f *.log > test.html
firefox test.html
---
tools/scraper/common.py | 19 +++++++++++++++++
tools/scraper/parser.py | 53 ++++++++++++++++++++++++++++++------------------
tools/scraper/scraper.py | 6 ++++++
3 files changed, 58 insertions(+), 20 deletions(-)
diff --git a/tools/scraper/common.py b/tools/scraper/common.py
index 8f2b9c0..cdd04ec 100755
--- a/tools/scraper/common.py
+++ b/tools/scraper/common.py
@@ -101,6 +101,10 @@ class Common():
# when --no-data is in effect, how many log lines were skipped?
data_skipped = 0
+ # List of router log module names to include verbatim.
+ # Defaults to "SCRAPER". Overridden by command line.
+ verbatim_include_list = ["SCRAPER"]
+
def router_id_index(self, id):
"""
Given a router full container name, return the index in router_ids table
@@ -110,6 +114,21 @@ class Common():
"""
return self.router_ids.index(id)
+ def module_key_in_line(self, key, line):
+ '''
+ Sense if the key is a log module name in the log line.
+ The name can't be too far into the string or else it finds
+ false positives when a user uses qdstat to get a log file.
+ MAX_POSITION defines what constitutes 'too far'.
+ :param key:
+ :param line:
+ :return:
+ '''
+ MAX_POSITION = 40
+ assert len(key) > 0
+ st = line.find(key)
+ return st >= 0 and st <= MAX_POSITION
+
def log_letter_of(idx):
'''
diff --git a/tools/scraper/parser.py b/tools/scraper/parser.py
index edf110a..c0c998a 100755
--- a/tools/scraper/parser.py
+++ b/tools/scraper/parser.py
@@ -389,7 +389,6 @@ class ParsedLogLine(object):
router_ls_key = "ROUTER_LS (info)"
transfer_key = "@transfer(20)"
proton_frame_key = "FRAME: "
- scraper_key = "SCRAPER ("
def sender_settle_mode_of(self, value):
if value == "0":
@@ -808,12 +807,18 @@ class ParsedLogLine(object):
:param _comn:
:param _router:
"""
- if not (ParsedLogLine.server_trace_key in _line or
- ParsedLogLine.protocol_trace_key in _line or
- (ParsedLogLine.policy_trace_key in _line and "lookup_user:" in _line) or # open (not begin, attach)
- ParsedLogLine.server_info_key in _line or
- ParsedLogLine.router_ls_key in _line or
- ParsedLogLine.scraper_key in _line):
+ verbatim_module = None
+ if len(_comn.verbatim_include_list) > 0:
+ for modx in _comn.verbatim_include_list:
+ if _comn.module_key_in_line(modx, _line):
+ verbatim_module = modx
+ break
+ if not (_comn.module_key_in_line(self.server_trace_key, _line) or
+ _comn.module_key_in_line(self.protocol_trace_key, _line) or
+ (_comn.module_key_in_line(self.policy_trace_key, _line) and "lookup_user:" in _line) or # open (not begin, attach)
+ _comn.module_key_in_line(self.server_info_key, _line) or
+ _comn.module_key_in_line(self.router_ls_key, _line) or
+ verbatim_module is not None):
raise ValueError("Line is not a candidate for parsing")
self.index = _log_index # router prefix 0 for A, 1 for B
self.instance = _instance # router instance in log file
@@ -877,17 +882,20 @@ class ParsedLogLine(object):
raise ValueError("Line too late outside time-of-day limits")
# Pull out scraper literal logs
- sti = self.line.find(self.scraper_key)
- if sti > 0:
- # strip datetime and show literal string
- sti += len("SCRAPER")
- self.data.is_scraper = True
- self.data.web_show_str = ("<strong>SCRAPER</strong> %s" % common.html_escape(self.line[sti:]))
- stcp = self.line[sti:].find(')') # close paren after log level
- if stcp < 0:
- stcp = 0
- self.data.sdorg_str = self.line[sti + stcp + 1:]
- return
+ if verbatim_module is not None:
+ sti = self.line.find(verbatim_module)
+ if sti > 0:
+ # strip datetime and show literal string
+ sti += len(verbatim_module)
+ self.data.is_scraper = True
+ self.data.web_show_str = ("<strong>%s</strong> %s" % (verbatim_module, common.html_escape(self.line[sti:])))
+ stcp = self.line[sti:].find(')') # close paren after log level
+ if stcp < 0:
+ stcp = 0
+ self.data.sdorg_str = self.line[sti + stcp + 1:]
+ return
+ else:
+ assert False # verbatim module was found only moments ago...
# extract connection number
sti = self.line.find(self.server_trace_key)
@@ -1013,7 +1021,6 @@ def parse_log_file(fn, log_index, comn):
keys = [key1, key3]
key4 = "ROUTER (info) Version:" # router version line
key5 = "ROUTER (info) Router started in " # router mode
- key6 = "SCRAPER (" # verbatim log lines to copy to Log Data
with open(fn, 'r') as infile:
for line in infile:
if search_for_in_progress:
@@ -1028,6 +1035,12 @@ def parse_log_file(fn, log_index, comn):
search_for_in_progress = False
rtr.restart_rec = router.RestartRecord(rtr, line, lineno + 1)
lineno += 1
+ verbatim_module = None
+ if len(comn.verbatim_include_list) > 0:
+ for modx in comn.verbatim_include_list:
+ if comn.module_key_in_line(modx, line):
+ verbatim_module = modx
+ break
if key2 in line:
# This line closes the current router, if any, and opens a new one
if rtr is not None:
@@ -1058,7 +1071,7 @@ def parse_log_file(fn, log_index, comn):
rtr.version = line[(line.find(key4) + len(key4)):].strip().split()[0]
elif key5 in line:
rtr.mode = line[(line.find(key5) + len(key5)):].strip().split()[0].lower()
- elif key6 in line:
+ elif verbatim_module is not None:
pl = ParsedLogLine(log_index, instance, lineno, line, comn, rtr)
rtr.lines.append(pl)
elif "[" in line and "]" in line:
diff --git a/tools/scraper/scraper.py b/tools/scraper/scraper.py
index c6a347f..cc618e6 100755
--- a/tools/scraper/scraper.py
+++ b/tools/scraper/scraper.py
@@ -103,6 +103,8 @@ def main_except(argv):
p.add_argument('--sequence', '-sq',
action='store_true',
help='Write sequence diagram raw data to end of web page. Edit what you need for input to seq-diag-gen utility.')
+ p.add_argument('--log-modules', '-lm',
+ help='Include list of named modules'' log entries in main log display. Specify multiple modules as a CSV string. Defaults to "SCRAPER"')
p.add_argument('--files', '-f', nargs="+")
del argv[0]
@@ -120,6 +122,10 @@ def main_except(argv):
except:
sys.exit("ERROR: Failed to parse time_end '%s'. Use format 'YYYY-MM-DD HH:MM:SS.n_uS'" % comn.args.time_end)
+ if not comn.args.log_modules is None:
+ l = [x.strip() for x in comn.args.log_modules.split(",")]
+ comn.verbatim_include_list = [x for x in l if len(x) > 0]
+
# process split function
if comn.args.split:
# Split processes only a single file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org
For additional commands, e-mail: commits-help@qpid.apache.org