You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@qpid.apache.org by ch...@apache.org on 2018/11/28 22:48:33 UTC
[2/2] qpid-dispatch git commit: DISPATCH-1191: Log scraper: fix
inefficiency when processing large logs
DISPATCH-1191: Log scraper: fix inefficiency when processing large logs
Create sub-lists of selected transfer frames and don't search whole list.
Fix links so transfer data is visible when message progress is suppressed.
Project: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/repo
Commit: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/commit/1a80b424
Tree: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/tree/1a80b424
Diff: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/diff/1a80b424
Branch: refs/heads/master
Commit: 1a80b4246b6c5127f2fc209494f47bdb9b2658d6
Parents: c0240d6
Author: Chuck Rolke <cr...@redhat.com>
Authored: Wed Nov 28 17:45:54 2018 -0500
Committer: Chuck Rolke <cr...@redhat.com>
Committed: Wed Nov 28 17:45:54 2018 -0500
----------------------------------------------------------------------
bin/log_scraper/main.py | 54 ++++++++++++++++++++-------------------
bin/log_scraper/nicknamer.py | 24 ++++++++++++++---
bin/log_scraper/parser.py | 4 +--
3 files changed, 50 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/1a80b424/bin/log_scraper/main.py
----------------------------------------------------------------------
diff --git a/bin/log_scraper/main.py b/bin/log_scraper/main.py
index c3891f2..344d2e1 100755
--- a/bin/log_scraper/main.py
+++ b/bin/log_scraper/main.py
@@ -222,6 +222,10 @@ def main_except(argv):
cn_v = comn.router_ids[common.index_of_log_letter(val)]
comn.conn_peers_display[key] = comn.shorteners.short_rtr_names.translate(cn_v)
comn.conn_peers_display[val] = comn.shorteners.short_rtr_names.translate(cn_k)
+
+ # sort transfer short name customer lists
+ comn.shorteners.short_data_names.sort_customers()
+
#
# Start producing the output stream
#
@@ -563,10 +567,9 @@ def main_except(argv):
for i in range(0, comn.shorteners.short_data_names.len()):
sname = comn.shorteners.short_data_names.shortname(i)
size = 0
- for plf in tree:
- if plf.data.name == "transfer" and plf.transfer_short_name == sname:
- size = plf.data.transfer_size
- break
+ for plf in comn.shorteners.short_data_names.customers(sname):
+ size = plf.data.transfer_size
+ break
print("<a name=\"%s\"></a> <h4>%s (%s)" % (sname, sname, size))
print(" <span> <a href=\"javascript:toggle_node('%s')\"> %s</a>" % ("data_" + sname, text.lozenge()))
print(" <div width=\"100%%\"; style=\"display:none; font-weight: normal; margin-bottom: 2px\" id=\"%s\">" %
@@ -580,28 +583,27 @@ def main_except(argv):
"<th>T delta</th> <th>T elapsed</th><th>Settlement</th><th>S elapsed</th></tr>")
t0 = None
tlast = None
- for plf in tree:
- if plf.data.name == "transfer" and plf.transfer_short_name == sname:
- if t0 is None:
- t0 = plf.datetime
- tlast = plf.datetime
- delta = "0.000000"
- epsed = "0.000000"
- else:
- delta = time_offset(plf.datetime, tlast)
- epsed = time_offset(plf.datetime, t0)
- tlast = plf.datetime
- sepsed = ""
- if plf.data.final_disposition is not None:
- sepsed = time_offset(plf.data.final_disposition.datetime, t0)
- rid = plf.router.iname
- peerconnid = "%s" % comn.conn_peers_connid.get(plf.data.conn_id, "")
- peer = plf.router.conn_peer_display.get(plf.data.conn_id, "") # peer container id
- print("<tr><td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> "
- "<td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> </tr>" %
- (plf.adverbl_link_to(), plf.datetime, rid, plf.data.conn_id, plf.data.direction,
- peerconnid, peer, delta, epsed,
- plf.data.disposition_display, sepsed))
+ for plf in comn.shorteners.short_data_names.customers(sname):
+ if t0 is None:
+ t0 = plf.datetime
+ tlast = plf.datetime
+ delta = "0.000000"
+ epsed = "0.000000"
+ else:
+ delta = time_offset(plf.datetime, tlast)
+ epsed = time_offset(plf.datetime, t0)
+ tlast = plf.datetime
+ sepsed = ""
+ if plf.data.final_disposition is not None:
+ sepsed = time_offset(plf.data.final_disposition.datetime, t0)
+ rid = plf.router.iname
+ peerconnid = "%s" % comn.conn_peers_connid.get(plf.data.conn_id, "")
+ peer = plf.router.conn_peer_display.get(plf.data.conn_id, "") # peer container id
+ print("<tr><td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> "
+ "<td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> </tr>" %
+ (plf.adverbl_link_to(), plf.datetime, rid, plf.data.conn_id, plf.data.direction,
+ peerconnid, peer, delta, epsed,
+ plf.data.disposition_display, sepsed))
print("</table>")
print("<hr>")
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/1a80b424/bin/log_scraper/nicknamer.py
----------------------------------------------------------------------
diff --git a/bin/log_scraper/nicknamer.py b/bin/log_scraper/nicknamer.py
index 9dc2f9f..f198270 100755
--- a/bin/log_scraper/nicknamer.py
+++ b/bin/log_scraper/nicknamer.py
@@ -19,6 +19,8 @@
# under the License.
#
+from collections import defaultdict
+import common
import cgi
class ShortNames():
@@ -34,8 +36,9 @@ class ShortNames():
self.longnames = []
self.prefix = prefixText
self.threshold = _threshold
+ self.customer_dict = defaultdict(list)
- def translate(self, lname, show_popup=False):
+ def translate(self, lname, show_popup=False, customer=None):
'''
Translate a long name into a short name, maybe.
Memorize all names, translated or not
@@ -52,12 +55,17 @@ class ShortNames():
self.longnames.append(lname)
idx = self.longnames.index(lname)
# return as-given if short enough
+ if customer is not None:
+ self.customer_dict[lname].append(customer)
if len(lname) < self.threshold:
return lname
+ sname = self.prefix + "_" + str(idx)
+ if customer is not None:
+ self.customer_dict[sname].append(customer)
if show_popup:
- return "<span title=\"" + cgi.escape(lname) + "\">" + self.prefix + "_" + str(idx) + "</span>"
+ return "<span title=\"" + cgi.escape(lname) + "\">" + sname + "</span>"
else:
- return self.prefix + "_" + str(idx)
+ return sname
def len(self):
return len(self.longnames)
@@ -104,11 +112,19 @@ class ShortNames():
print ("<ul>")
for i in range(0, len(self.longnames)):
name = self.prefix + "_" + str(i)
+ dump_anchor = "<a name=\"%s_dump\"></a>" % (name)
if with_link:
name = "<a href=\"#%s\">%s</a>" % (name, name)
- print ("<li> " + name + " - " + cgi.escape(self.longnames[i]) + "</li>")
+ print ("<li> " + dump_anchor + name + " - " + cgi.escape(self.longnames[i]) + "</li>")
print ("</ul>")
+ def sort_customers(self):
+ for c in common.dict_iterkeys(self.customer_dict):
+ l = self.customer_dict[c]
+ self.customer_dict[c] = sorted(l, key=lambda lfl: lfl.datetime)
+
+ def customers(self, sname):
+ return self.customer_dict[sname]
class Shorteners():
def __init__(self):
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/1a80b424/bin/log_scraper/parser.py
----------------------------------------------------------------------
diff --git a/bin/log_scraper/parser.py b/bin/log_scraper/parser.py
index 6d29356..895db65 100755
--- a/bin/log_scraper/parser.py
+++ b/bin/log_scraper/parser.py
@@ -495,8 +495,8 @@ class ParsedLogLine(object):
res.transfer_more = resdict.get("more", "") == "true"
res.transfer_resume = resdict.get("resume", "") == "true"
res.transfer_aborted = resdict.get("aborted", "") == "true"
- self.transfer_short_name = self.shorteners.short_data_names.translate(res.transfer_bare)
- showdat = "<a href=\"#%s\">%s</a>" % (self.transfer_short_name, self.transfer_short_name)
+ self.transfer_short_name = self.shorteners.short_data_names.translate(res.transfer_bare, customer=self)
+ showdat = "<a href=\"#%s_dump\">%s</a>" % (self.transfer_short_name, self.transfer_short_name)
res.web_show_str = "<strong>%s</strong> %s (%s) %s %s %s %s %s - %s bytes" % (
res.name, colorize_bg(res.channel_handle), res.delivery_id,
self.highlighted("settled", res.transfer_settled, "green"),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org
For additional commands, e-mail: commits-help@qpid.apache.org