You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@qpid.apache.org by ch...@apache.org on 2020/04/06 20:31:07 UTC
[qpid-dispatch] branch master updated: DISPATCH-1616: Add Scraper
tooling to export source for sequence diagrams
This is an automated email from the ASF dual-hosted git repository.
chug pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/qpid-dispatch.git
The following commit(s) were added to refs/heads/master by this push:
new fad96d5 DISPATCH-1616: Add Scraper tooling to export source for sequence diagrams
fad96d5 is described below
commit fad96d5ccb73fe81c0ddc7f168583a41a3d52410
Author: Chuck Rolke <ch...@apache.org>
AuthorDate: Mon Apr 6 16:30:26 2020 -0400
DISPATCH-1616: Add Scraper tooling to export source for sequence diagrams
---
tools/scraper/README.md | 27 +++++++
tools/scraper/parser.py | 23 ++++++
tools/scraper/scraper.py | 29 ++++++++
tools/scraper/seq-diag-gen.py | 164 ++++++++++++++++++++++++++++++++++++++++++
4 files changed, 243 insertions(+)
diff --git a/tools/scraper/README.md b/tools/scraper/README.md
index 093fd92..f5fde0a 100644
--- a/tools/scraper/README.md
+++ b/tools/scraper/README.md
@@ -156,6 +156,33 @@ and divides the connection data. Connection names inclued the router instance ID
* AMQP Addresses from every every AMQP Attach are indexed. A table for each address
shows when the address was referenced and some connection details.
+## Sequence generator
+
+The sequence generator (switches --sequence or -sq) generates some extra output that is
+useful for making sequence diagrams from the raw data. _This is an experimental feature._
+
+When a sequence switch is set then the AMQP performatives are simplified and dumped
+into the html data stream near the end.
+
+A user then identifies what parts of the log data he needs and then edits the lines
+in question from the html data stream file. Careful attention to the start and
+end timestamps will help a user find the data more easily. Save the selected data into an
+intermediate file.
+
+Finally the intermediate file is processed by new file _tools/scraper/seq-diag-gen.py_
+to produce the sequence diagram source file.
+
+Items to do to make this a reliable feature:
+
+* In most cases the the non-instantaneous messages do not exactly line up on a precisely
+between the sender's and receiver's time lines.
+* The vertical time scale is wildly exaggerated. There is no scale: equal intervals in
+vertical space could be microseconds or days.
+* Messages from non-router actors are shown as horizontal lines. This tool has no access
+to those actors' trace logs or timestamps.
+* You may want to manually specify actors and participants to get the vertical time lines
+in an acceptable order.
+
## Quick Start
* Enable router logging
diff --git a/tools/scraper/parser.py b/tools/scraper/parser.py
index 12e89b2..135dbc7 100755
--- a/tools/scraper/parser.py
+++ b/tools/scraper/parser.py
@@ -99,6 +99,7 @@ class LogLineData:
def __init__(self):
self.web_show_str = ""
+ self.sdorg_str = "" # text for sequence diagram source
self.name = ""
self.conn_num = "" # source router's undecorated conn num
self.conn_id = "" # decorated routerPrefixLetter'instanceNumber-conn_num
@@ -410,10 +411,14 @@ class ParsedLogLine(object):
result = "<span style=\"background-color:%s\">%s</span>" % (color, name)
return result
+ def unhighlighted(self, name, value):
+ return name if value else ""
+
def extract_facts(self):
perf = self.data.described_type.dtype_number
res = self.data
resdict = self.data.described_type.dict
+ res.sdorg_str = "HELP I'M A ROCK - Unknown performative: %s" % perf
# the performatives
# Note: res.channel is already populated
@@ -422,9 +427,11 @@ class ParsedLogLine(object):
res.name = "open"
res.channel = "0"
res.web_show_str = "<strong>%s</strong> [%s]" % (res.name, res.channel)
+ res.sdorg_str = "OPEN"
if res.direction == text.direction_in():
res.conn_peer = self.resdict_value(resdict, "container-id", "unknown")
res.web_show_str += (" (peer: %s)" % res.conn_peer)
+ res.sdorg_str += (" (peer: %s)" % res.conn_peer)
elif perf == 0x11:
# Performative: begin [channel,remoteChannel]
@@ -435,6 +442,7 @@ class ParsedLogLine(object):
res.remote = self.resdict_value(resdict, "remote-channel", "None)")
res.channel_remote = "[%s,%s]" % (res.channel, res.remote)
res.web_show_str = "<strong>%s</strong> %s" % (res.name, res.channel_remote)
+ res.sdorg_str = "BEGIN %s" % (res.channel_remote)
elif perf == 0x12:
# Performative: attach [channel,handle] role name (source: src, target: tgt)
@@ -490,6 +498,7 @@ class ParsedLogLine(object):
res.web_show_str = "<strong>%s</strong> %s (%s,%s) %s" % (
res.name, colorize_bg(res.channel_handle), res.flow_deliverycnt, res.flow_linkcredit,
self.highlighted("drain", res.flow_drain, common.color_of("drain")))
+ res.sdorg_str = "FLOW %s (%s,%s)" % (res.channel_handle, res.flow_deliverycnt, res.flow_linkcredit) # TODO: Add drain
elif perf == 0x14:
# Performative: transfer [channel,handle] (id)
@@ -521,6 +530,8 @@ class ParsedLogLine(object):
### colorize_dispositions_not_accepted(proto, res, global_vars, count_anomalies)
res.web_show_str = ("<strong>%s</strong> [%s] (%s %s-%s settled=%s state=%s)" %
(res.name, res.channel, res.role, res.first, res.last, res.settled, res.disposition_state))
+ res.sdorg_str = ("%s [%s] (%s %s-%s settled=%s state=%s)" %
+ (res.name, res.channel, res.role, res.first, res.last, res.settled, res.disposition_state))
elif perf == 0x16:
# Performative: detach [channel, handle]
@@ -529,12 +540,14 @@ class ParsedLogLine(object):
### TODO: colorize_performative_error(proto, res, global_vars, count_anomalies)
res.channel_handle = "[%s,%s]" % (res.channel, res.handle)
res.web_show_str = "<strong>%s</strong> %s" % (res.name, colorize_bg(res.channel_handle))
+ res.sdorg_str = "DETACH %s" % (res.channel_handle)
elif perf == 0x17:
# Performative: end [channel]
res.name = "end"
### TODO: colorize_performative_error(proto, res, global_vars, count_anomalies)
res.web_show_str = "<strong>%s</strong> [%s]" % (res.name, res.channel)
+ res.sdorg_str = "END [%s]" % (res.channel)
elif perf == 0x18:
# Performative: close [0] always channel 0
@@ -542,6 +555,7 @@ class ParsedLogLine(object):
res.name = "close"
### colorize_performative_error(proto, res, global_vars, count_anomalies)
res.web_show_str = "<strong>%s</strong> [%s]" % (res.name, res.channel)
+ res.sdorg_str = "CLOSE [%s]" % (res.channel)
elif perf == 0x1d:
# transport:definitions error
@@ -725,6 +739,9 @@ class ParsedLogLine(object):
res.web_show_str = ("<strong>%s</strong> %s %s %s (source: %s, target: %s, class: %s)" %
(res.name, colorize_bg(res.channel_handle), res.role, res.link_short_name_popup,
res.source, res.target, res.link_class))
+ res.sdorg_str = ("%s %s %s %s (src: %s, tgt: %s)" %
+ (res.name, res.channel_handle, res.role, res.link_short_name,
+ res.source, res.target))
elif perf == 0x14:
# Performative: transfer [channel,handle] (id)
self.transfer_short_name = self.shorteners.short_data_names.translate(res.transfer_bare, customer=self)
@@ -736,6 +753,12 @@ class ParsedLogLine(object):
self.highlighted("resume", res.transfer_resume, common.color_of("aborted")),
self.highlighted("aborted", res.transfer_aborted, common.color_of("aborted")),
showdat, res.transfer_size)
+ res.sdorg_str = "%s %s (%s) %s %s%s%s%s" % (
+ res.name, res.channel_handle, res.delivery_id, res.transfer_size,
+ self.unhighlighted(" settled", res.transfer_settled),
+ self.unhighlighted(" more", res.transfer_more),
+ self.unhighlighted(" resume", res.transfer_resume),
+ self.unhighlighted(" aborted", res.transfer_aborted))
def adverbl_link_to(self):
"""
diff --git a/tools/scraper/scraper.py b/tools/scraper/scraper.py
index 5061129..8e21277 100755
--- a/tools/scraper/scraper.py
+++ b/tools/scraper/scraper.py
@@ -100,6 +100,9 @@ def main_except(argv):
help='Ignore log records earlier than this. Format: "2018-08-13 13:15:00.123456"')
p.add_argument('--time-end', '-te',
help='Ignore log records later than this. Format: "2018-08-13 13:15:15.123456"')
+ p.add_argument('--sequence', '-sq',
+ action='store_true',
+ help='Write sequence diagram raw data to end of web page. Edit what you need for input to seq-diag-gen utility.')
p.add_argument('--files', '-f', nargs="+")
del argv[0]
@@ -923,6 +926,32 @@ def main_except(argv):
print("<hr>")
+
+ # Emit data for source to be processed by seq-diag-gen utility
+ if comn.args.sequence:
+ print("<h3>sequencediagram.org data</h3>")
+ for plf in tree:
+ rtr = plf.router
+ rid = comn.router_display_names[rtr.log_index]
+ peer = rtr.conn_peer_display.get(plf.data.conn_id, "")
+ if peer.startswith("<"):
+ peer = peer[peer.find(">")+1:]
+ peer = peer[:peer.find("<")]
+ # TODO: differentiate between peer sender and peer receiver
+ # This is not so easy as test code uses the same container id for all connections
+ # Easiest thing to do is append peer with router_id. Does not fix tests with sender/receiver
+ # to same router.
+ if peer.startswith("peer"):
+ peer += "_" + rid
+ #TODO handle EOS, connection open/close lines
+ if (not plf.data.sdorg_str == "" and
+ not plf.data.direction == "" and
+ not plf.data.sdorg_str.startswith("HELP")):
+ print("%s|%s|%s|%s|%s|%s" % (plf.datetime, rid, plf.data.direction, peer, plf.data.sdorg_str, ("%s#%d" % (plf.prefixi, plf.lineno))))
+ #import pdb
+ #pdb.set_trace()
+ print("<hr>")
+
print("</body>")
diff --git a/tools/scraper/seq-diag-gen.py b/tools/scraper/seq-diag-gen.py
new file mode 100755
index 0000000..7313f8e
--- /dev/null
+++ b/tools/scraper/seq-diag-gen.py
@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from typing import List
+import optparse
+import fileinput
+import json
+import re
+import sys
+
+"""
+This program accepts input for a modified Scraper and produces output to create sequence diagrams
+at sequencediagram.org.
+"""
+
+IDX_DATETIME = 0
+IDX_NAME_LEFT = 1
+IDX_DIR = 2
+IDX_NAME_RIGHT = 3
+IDX_PERFORMATIVE = 4
+IDX_ROUTER_LINE = 5
+
+# Exactly how to predict how much 'space' asynchronous messages take is vague.
+MAGIC_SPACE_NUMBER = 1 # tested with entryspacing 0.1
+
+class log_record:
+ def __init__(self, index, line):
+ # print("DEBUG input line: ", index, line)
+ dateandtime, name_left, direction, name_right, perf, router_line = line.split('|')
+ self.dateandtime = dateandtime.strip()
+ self.index = index
+ self.name_left = name_left.strip()
+ self.direction = direction.strip()
+ self.name_right = name_right.strip()
+ self.performative = perf.strip()
+ self.router_line = router_line.strip() # diag
+ self.peer_log_rec = None
+ dir_out = (direction == '->') # name_left -> name_right
+ self.sentby = name_left.strip() if dir_out else name_right.strip()
+ self.rcvdby = name_right.strip() if dir_out else name_left.strip()
+
+ def absorb_peer_rec(self, peer_rec):
+ '''
+ Given peer_rec, see if it is the receiving side of
+ this log_rec being sent. If so then cross-link the
+ records and return true.
+ :param peer_rec:
+ :return:
+ '''
+ valid_peer = self.name_left == peer_rec.name_right and \
+ self.name_right == peer_rec.name_left and \
+ self.direction != peer_rec.direction and \
+ self.performative.startswith(peer_rec.performative) and \
+ self.peer_log_rec is None and \
+ peer_rec.peer_log_rec is None
+ if not valid_peer:
+ return False
+ self.peer_log_rec = peer_rec
+ peer_rec.peer_log_rec = self
+ return True
+
+ def show_for_sdorg(self):
+ # A single sd.org vector consumes one or two log lines.
+ # One if from an external actor.
+ # Two if a message goes between routers.
+ # Leave space where the message is supposed to land in the receiving routers
+ if self.peer_log_rec is None:
+ print("%s->%s:%s" % (self.sentby, self.rcvdby, self.performative))
+ else:
+ if self.peer_log_rec.index > self.index:
+ linediff = int(self.peer_log_rec.index) - int(self.index)
+ space = -MAGIC_SPACE_NUMBER * (linediff - 1)
+ print("%s->(%s)%s:%s" %(self.sentby, str(linediff), self.rcvdby, self.performative))
+ print("space %s" % (str(space)))
+ else:
+ print("space %d" % MAGIC_SPACE_NUMBER)
+
+ def diag_dump(self):
+ cmn = ("index: %d, sentby: %s, rcvdby: %s, performative: %s, router_line: %s" %
+ (self.index, self.sentby, self.rcvdby, self.performative, self.router_line))
+ if self.peer_log_rec is None:
+ print(cmn)
+ else:
+ print("%s, PEER MATCH peer_index: %s, peer_router_log_line: %s" %
+ (cmn, self.peer_log_rec.index, self.peer_log_rec.router_line))
+
+
+def split_log_file(filename):
+ '''
+ Given a filename, read the file into an array of lines
+ :param filename:
+ :return:
+ '''
+ with open(logfile, 'r') as log:
+ log_lines = log.read().split("\n")
+ return log_lines
+
+
+def match_logline_pairs(log_recs):
+ '''
+ Given a log line that might be the source of a message to another router,
+ search the list of remaining log lines and try to find the router that received
+ this message. Then hook them together.
+ The cut of Scraper output presented as input to this program may not be complete.
+ Messages for which there is no 'match' are drawn as horizontal lines.
+ :param log_lines:
+ :return:
+ '''
+ for idx in range(len(log_recs)-2):
+ for idx2 in range(idx+1, len(log_recs)-1):
+ if log_recs[idx].absorb_peer_rec(log_recs[idx2]):
+ break
+
+
+if __name__ == "__main__":
+ parser = optparse.OptionParser(usage="%prog [options]",
+ description="cooks a scraper log snippet into sequencediagram.org source")
+ parser.add_option("-f", "--filename", action="append", help="logfile to use or - for stdin")
+
+ (opts, args) = parser.parse_args()
+ if not opts.filename:
+ opts.filename = ["-"]
+
+ log_recs = []
+ for logfile in opts.filename:
+ # print("Parsing: %s" % logfile)
+ log_lines = split_log_file(logfile)
+ if len(log_lines) >= 2:
+ index = 0
+ for logline in log_lines:
+ if len(logline) > 0:
+ log_recs.append( log_record(index, logline) )
+ index += 1
+ match_logline_pairs(log_recs)
+
+ # process the list of records
+ for log_rec in log_recs:
+ log_rec.show_for_sdorg()
+
+ # diag dump
+ #print("\nDiag dump")
+ #for log_rec in log_recs:
+ # log_rec.diag_dump()
+ else:
+ print("Log file has fewer than two lines. I give up.")
+ print()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org
For additional commands, e-mail: commits-help@qpid.apache.org