You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@qpid.apache.org by ch...@apache.org on 2018/12/13 20:12:17 UTC
qpid-dispatch git commit: DISPATCH-1216: Scraper improve --split
performance and display
Repository: qpid-dispatch
Updated Branches:
refs/heads/master 16674f44b -> b7f7b13c7
DISPATCH-1216: Scraper improve --split performance and display
* use lightweight parser for attach lines
* sort address tables alphabetically
* show/hide gizmos for address tables
Project: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/repo
Commit: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/commit/b7f7b13c
Tree: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/tree/b7f7b13c
Diff: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/diff/b7f7b13c
Branch: refs/heads/master
Commit: b7f7b13c71b82e744aec941e2de0638999283047
Parents: 16674f4
Author: Chuck Rolke <cr...@redhat.com>
Authored: Thu Dec 13 15:07:54 2018 -0500
Committer: Chuck Rolke <cr...@redhat.com>
Committed: Thu Dec 13 15:07:54 2018 -0500
----------------------------------------------------------------------
tools/scraper/log_splitter.py | 117 ++++++++++++++++++++++++++++++++-----
1 file changed, 102 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/b7f7b13c/tools/scraper/log_splitter.py
----------------------------------------------------------------------
diff --git a/tools/scraper/log_splitter.py b/tools/scraper/log_splitter.py
index 0a77182..30ffc76 100755
--- a/tools/scraper/log_splitter.py
+++ b/tools/scraper/log_splitter.py
@@ -30,13 +30,13 @@ from __future__ import absolute_import
from __future__ import print_function
import cgi
+from datetime import *
import os
import sys
import traceback
from collections import defaultdict
import common
-import parser
import text
class connection():
@@ -69,6 +69,86 @@ class connection():
self.path_name = self.log_n_dir + "/" + self.file_name
+class parsed_attach():
+ """
+ Parse an attach log line. The usual parser is way too slow
+ so this does the essentials for --split.
+ """
+ def find_field(self, key, line):
+ sti = line.find(key)
+ if sti < 0:
+ return 'none'
+ ste = line.find(',', sti + len(key))
+ if ste < 0:
+ raise ValueError("Value not properly delimited. Key '%s'. line: %s" % (key, self.line))
+ val = line[sti + len(key):ste]
+ if val.startswith('"'):
+ val = val[1:-1]
+ return val
+
+
+ def __init__(self, instance, line, opaque):
+ self.instance = instance
+ self.line = line
+ self.opaque = opaque
+ self.datetime = None
+ self.conn_num = ""
+ self.conn_id = ""
+ self.direction = ""
+ self.role = ""
+ self.source = ""
+ self.target = ""
+
+ # timestamp
+ try:
+ self.datetime = datetime.strptime(self.line[:26], '%Y-%m-%d %H:%M:%S.%f')
+ except:
+ # old routers flub the timestamp and don't print leading zero in uS time
+ # 2018-11-18 11:31:08.269 should be 2018-11-18 11:31:08.000269
+ td = self.line[:26]
+ parts = td.split('.')
+ us = parts[1]
+ parts_us = us.split(' ')
+ if len(parts_us[0]) < 6:
+ parts_us[0] = '0' * (6 - len(parts_us[0])) + parts_us[0]
+ parts[1] = ' '.join(parts_us)
+ td = '.'.join(parts)
+ try:
+ self.datetime = datetime.strptime(td[:26], '%Y-%m-%d %H:%M:%S.%f')
+ except:
+ self.datetime = datetime(1970, 1, 1)
+ key_strace = "SERVER (trace) ["
+ sti = self.line.find(key_strace)
+ if sti < 0:
+ raise ValueError("'%s' not found in line %s" % (key_strace, self.line))
+ self.line = self.line[sti + len(key_strace):]
+ ste = self.line.find(']')
+ if ste < 0:
+ print("Failed to parse line ", self.line)
+ raise ValueError("'%s' not found in line %s" % ("]", self.line))
+ self.conn_num = self.line[:ste]
+ self.line = self.line[ste + 1:]
+ self.conn_id = "A" + str(self.instance) + "_" + str(self.conn_num)
+ # get the session (channel) number
+ if self.line.startswith(':'):
+ self.line = self.line[1:]
+ sti = self.line.find(' ')
+ if sti < 0:
+ raise ValueError("space not found after channel number at head of line %s" % (self.line))
+ if sti > 0:
+ self.channel = self.line[:sti]
+ self.line = self.line[sti + 1:]
+ self.line = self.line.lstrip()
+ # direction
+ if self.line.startswith('<') or self.line.startswith('-'):
+ self.direction = self.line[:2]
+ self.line = self.line[3:]
+ else:
+ raise ValueError("line does not have direction arrow: %s" % (self.line))
+ self.role = "receiver" if self.find_field('role=', self.line) == "true" else "sender"
+ self.source = self.find_field('@source(40) [address=', self.line)
+ self.target = self.find_field('@target(41) [address=', self.line)
+
class LogFile:
def __init__(self, fn, top_n=24):
"""
@@ -455,7 +535,7 @@ function show_node(node)
for k, conn in dict_iteritems(self.connections):
for aline in conn.attaches:
try:
- pl = parser.ParsedLogLine(0, conn.instance, 0, aline, comn, None, k)
+ pl = parsed_attach(conn.instance, aline, k)
except Exception as e:
# t, v, tb = sys.exc_info()
if hasattr(e, 'message'):
@@ -463,20 +543,17 @@ function show_node(node)
else:
sys.stderr.write("Failed to parse %s. Analysis continuing...\n" % (e))
if pl is not None:
- nn2[pl.data.source].append(pl)
- if pl.data.source != pl.data.target:
- nn2[pl.data.target].append(pl)
+ nn2[pl.source].append(pl)
+ if pl.source != pl.target:
+ nn2[pl.target].append(pl)
print("<h3>Verbose AMQP Addresses Overview (N=%d)</h3>" % len(nn2))
- showthis = ("<a href=\"javascript:toggle_node('addr_table_2')\">%s</a>" %
- (text.lozenge()))
- print(" %s This table shows addresses that referenced in Attach performatives. <br>" % showthis)
- print("<div id=\"addr_table_2\" style=\"display:none; margin-top: 2px; margin-bottom: 2px; margin-left: 10px\">")
addr_many = []
addr_few = []
ADDR_LEVEL = 4
n = 0
- for k, plfs in dict_iteritems(nn2):
+ for k in sorted(nn2.keys()):
+ plfs = nn2[k]
showthis = ("<a href=\"javascript:toggle_node('@@addr2_%d')\">%s</a>" %
(n, text.lozenge()))
visitthis = ("<a href=\"#@@addr2_%d_data\">%s</a>" %
@@ -488,20 +565,31 @@ function show_node(node)
else:
addr_many.append(line)
n += 1
+ showthis = ("<a href=\"javascript:toggle_node('addr_table_many')\">%s</a>" %
+ (text.lozenge()))
+ print(" %s Addresses attached more than %d times (N=%d) <br>" % (showthis, ADDR_LEVEL, len(addr_many)))
+ print("<div id=\"addr_table_many\" style=\"display:none; margin-top: 2px; margin-bottom: 2px; margin-left: 10px\">")
print("<h4>Addresses with many links (N=%d)</h4>" % (len(addr_many)))
print("<table><tr> <th>Address</th> <th>N References</th> </tr>")
for line in addr_many: print(line)
print("</table>")
+ print("</div>")
+ showthis = ("<a href=\"javascript:toggle_node('addr_table_few')\">%s</a>" %
+ (text.lozenge()))
+ print(" %s Addresses attached %d times or fewer (N=%d)<br>" % (showthis, ADDR_LEVEL, len(addr_few)))
+ print("<div id=\"addr_table_few\" style=\"display:none; margin-top: 2px; margin-bottom: 2px; margin-left: 10px\">")
print("<h4>Addresses with few links (N=%d)</h4>" % (len(addr_few)))
print("<table><tr> <th>Address</th> <th>N References</th> </tr>")
for line in addr_few: print(line)
print("</table>")
+ print("</div>")
# loop to print expandable sub tables
print("<h3>AMQP Addresses Details</h3>")
n = 0
- for k, plfss in dict_iteritems(nn2):
+ for k in sorted(nn2.keys()):
+ plfss = nn2[k]
plfs = sorted(plfss, key=lambda lfl: lfl.datetime)
print("<div id=\"@@addr2_%d\" style=\"display:none; margin-top: 2px; margin-bottom: 2px; margin-left: 10px\">" %
(n))
@@ -510,13 +598,12 @@ function show_node(node)
print("<table><tr><th>Time</th> <th>Connection</th> <th>Dir</th> <th>Peer</th> <th>Role</th> <th>Source</th> <th>Target</th> </tr>")
for plf in plfs:
print("<tr><td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> </tr>" %
- (plf.datetime, plf.data.conn_id,
- plf.data.direction, self.connections[plf.opaque].peer_type,
- plf.data.role, plf.data.source, plf.data.target))
+ (plf.datetime, plf.conn_id,
+ plf.direction, self.connections[plf.opaque].peer_type,
+ plf.role, plf.source, plf.target))
print("</table>")
print("</div>")
n += 1
- print("</div>")
# py 2-3 compat
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org
For additional commands, e-mail: commits-help@qpid.apache.org