You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@qpid.apache.org by ch...@apache.org on 2018/11/30 20:19:09 UTC
[5/7] qpid-dispatch git commit: DISPATCH-1199: move scraper tool to
tools/scraper directory
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/b7ab3390/bin/log_scraper/scraper.py
----------------------------------------------------------------------
diff --git a/bin/log_scraper/scraper.py b/bin/log_scraper/scraper.py
deleted file mode 100755
index 344d2e1..0000000
--- a/bin/log_scraper/scraper.py
+++ /dev/null
@@ -1,825 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# Adverbl concepts
-# * Multiple log files may be displayed at the same time.
-# Each log file gets a letter prefix: A, B, C, ...
-# * Log AMQP proton trace channel numbers get prefix
-# [1] becomes [A-1]
-# * The log file line numbers are equivalent to a wireshark trace frame number.
-# * There's no concept of client and server because the logs are from inside
-# a router.
-
-from __future__ import unicode_literals
-from __future__ import division
-from __future__ import absolute_import
-from __future__ import print_function
-
-import argparse
-import ast
-import cgi
-import os
-import sys
-import traceback
-
-import common
-import datetime
-from log_splitter import main_except as splitter_main
-import parser
-import router
-import text
-
-
-def time_offset(ttest, t0):
- """
- Return a string time delta between two datetime objects in seconds formatted
- to six significant decimal places.
- :param ttest:
- :param t0:
- :return:
- """
- delta = ttest - t0
- t = float(delta.seconds) + float(delta.microseconds) / 1000000.0
- return "%0.06f" % t
-
-
-def show_noteworthy_line(plf, comn):
- """
- Given a log line, print the noteworthy display line
- :param plf: parsed log line
- :param comn:
- :return:
- """
- rid = plf.router.iname
- id = "[%s]" % plf.data.conn_id
- peerconnid = "[%s]" % comn.conn_peers_connid.get(plf.data.conn_id, "")
- peer = plf.router.conn_peer_display.get(plf.data.conn_id, "") # peer container id
- print("%s %s %s %s %s %s %s<br>" %
- (plf.adverbl_link_to(), rid, id, plf.data.direction, peerconnid, peer,
- plf.data.web_show_str))
-
-
-#
-#
-def main_except(argv):
- # Instantiate a common block
- comn = common.Common()
-
- # optparse - look for data-inhibit and program mode control
- p = argparse.ArgumentParser()
- p.add_argument('--skip-all-data', '-sa',
- action='store_true',
- help='Max load shedding: do not store/index transfer, disposition, flow or EMPTY_FRAME data')
- p.add_argument('--skip-detail', '-sd',
- action='store_true',
- help='Load shedding: do not produce Connection Details tables')
- p.add_argument('--skip-msg-progress', '-sm',
- action='store_true',
- help='Load shedding: do not produce Message Progress tables')
- p.add_argument('--split', '-sp',
- action='store_true',
- help='A single file is split into per-connection data.')
- p.add_argument('--time-start', '-ts',
- help='Ignore log records earlier than this. Format: "2018-08-13 13:15:00.123456"')
- p.add_argument('--time-end', '-te',
- help='Ignore log records later than this. Format: "2018-08-13 13:15:15.123456"')
- p.add_argument('--files', '-f', nargs="+")
-
- del argv[0]
- comn.args = p.parse_args(argv)
-
- if not comn.args.time_start is None:
- try:
- comn.args.time_start = datetime.datetime.strptime(comn.args.time_start, "%Y-%m-%d %H:%M:%S.%f")
- except:
- sys.exit("ERROR: Failed to parse time_start '%s'. Use format 'YYYY-MM-DD HH:MM:SS.n_uS'" % comn.args.time_start)
-
- if not comn.args.time_end is None:
- try:
- comn.args.time_end = datetime.datetime.strptime(comn.args.time_end, "%Y-%m-%d %H:%M:%S.%f")
- except:
- sys.exit("ERROR: Failed to parse time_end '%s'. Use format 'YYYY-MM-DD HH:MM:SS.n_uS'" % comn.args.time_end)
-
- # process split function
- if comn.args.split:
- # Split processes only a single file
- if len(comn.args.files) > 1:
- sys.exit('--split mode takes only one file name')
- return splitter_main(comn.args.files[0])
-
- # process the log files and add the results to router_array
- for log_i in range(len(comn.args.files)):
- arg_log_file = comn.args.files[log_i]
- comn.log_fns.append(arg_log_file)
- comn.n_logs += 1
-
- if not os.path.exists(arg_log_file):
- sys.exit('ERROR: log file %s was not found!' % arg_log_file)
-
- # parse the log file
- rtrs = parser.parse_log_file(arg_log_file, log_i, comn)
- comn.routers.append(rtrs)
-
- # marshall facts about the run
- for rtr in rtrs:
- rtr.discover_connection_facts(comn)
-
- # Create lists of various things sorted by time
- tree = [] # log line
- ls_tree = [] # link state lines
- rr_tree = [] # restart records
- for rtrlist in comn.routers:
- for rtr in rtrlist:
- tree += rtr.lines
- ls_tree += rtr.router_ls
- rr_tree.append(rtr.restart_rec)
- tree = sorted(tree, key=lambda lfl: lfl.datetime)
- ls_tree = sorted(ls_tree, key=lambda lfl: lfl.datetime)
- rr_tree = sorted(rr_tree, key=lambda lfl: lfl.datetime)
-
- # Back-propagate a router name/version/mode to each list's router0.
- # Complain if container name or version changes between instances.
- # Fill in container_id and shortened display_name tables
- for fi in range(comn.n_logs):
- rtrlist = comn.routers[fi]
- if len(rtrlist) > 1:
- if rtrlist[0].container_name is None:
- rtrlist[0].container_name = rtrlist[1].container_name
- if rtrlist[0].version is None:
- rtrlist[0].version = rtrlist[1].version
- if rtrlist[0].mode is None:
- rtrlist[0].mode = rtrlist[1].mode
- for i in range(0, len(rtrlist) - 1):
- namei = rtrlist[i].container_name
- namej = rtrlist[i + 1].container_name
- if namei != namej:
- sys.exit('Inconsistent container names, log file %s, instance %d:%s but instance %d:%s' %
- (comn.log_fns[fi], i, namei, i + 1, namej))
- namei = rtrlist[i].version
- namej = rtrlist[i + 1].version
- if namei != namej:
- sys.exit('Inconsistent router versions, log file %s, instance %d:%s but instance %d:%s' %
- (comn.log_fns[fi], i, namei, i + 1, namej))
- namei = rtrlist[i].mode
- namej = rtrlist[i + 1].mode
- if namei != namej:
- sys.exit('Inconsistent router modes, log file %s, instance %d:%s but instance %d:%s' %
- (comn.log_fns[fi], i, namei, i + 1, namej))
- name = rtrlist[0].container_name if len(rtrlist) > 0 and rtrlist[0].container_name is not None else ("Unknown_%d" % fi)
- mode = rtrlist[0].mode if len(rtrlist) > 0 and rtrlist[0].mode is not None else "standalone"
- comn.router_ids.append(name)
- comn.router_display_names.append(comn.shorteners.short_rtr_names.translate(name))
- comn.router_modes.append(mode)
-
- # aggregate connection-to-frame maps into big map
- for rtrlist in comn.routers:
- for rtr in rtrlist:
- comn.conn_to_frame_map.update(rtr.conn_to_frame_map)
-
- # generate router-to-router connection peer relationships
- peer_list = []
- for plf in tree:
- if plf.data.name == "open" and plf.data.direction_is_in():
- cid = plf.data.conn_id # the router that generated this log file
- if "properties" in plf.data.described_type.dict:
- peer_conn = plf.data.described_type.dict["properties"].get(':"qd.conn-id"',
- "") # router that sent the open
- if peer_conn != "" and plf.data.conn_peer != "":
- pid_peer = plf.data.conn_peer.strip('\"')
- rtr, rtridx = router.which_router_id_tod(comn.routers, pid_peer, plf.datetime)
- if rtr is not None:
- pid = rtr.conn_id(peer_conn)
- hit = sorted((cid, pid))
- if hit not in peer_list:
- peer_list.append(hit)
-
- for (key, val) in peer_list:
- if key in comn.conn_peers_connid:
- sys.exit('key val messed up')
- if val in comn.conn_peers_connid:
- sys.exit('key val messed up')
- comn.conn_peers_connid[key] = val
- comn.conn_peers_connid[val] = key
- cn_k = comn.router_ids[common.index_of_log_letter(key)]
- cn_v = comn.router_ids[common.index_of_log_letter(val)]
- comn.conn_peers_display[key] = comn.shorteners.short_rtr_names.translate(cn_v)
- comn.conn_peers_display[val] = comn.shorteners.short_rtr_names.translate(cn_k)
-
- # sort transfer short name customer lists
- comn.shorteners.short_data_names.sort_customers()
-
- #
- # Start producing the output stream
- #
- print(text.web_page_head())
-
- #
- # Generate javascript
- #
- # output the frame show/hide functions into the header
- for conn_id, plfs in common.dict_iteritems(comn.conn_to_frame_map):
- print("function show_%s() {" % conn_id)
- for plf in plfs:
- print(" javascript:show_node(\'%s\');" % plf.fid)
- print("}")
- print("function hide_%s() {" % conn_id)
- for plf in plfs:
- print(" javascript:hide_node(\'%s\');" % plf.fid)
- print("}")
- # manipulate checkboxes
- print("function show_if_cb_sel_%s() {" % conn_id)
- print(" if (document.getElementById(\"cb_sel_%s\").checked) {" % conn_id)
- print(" javascript:show_%s();" % conn_id)
- print(" } else {")
- print(" javascript:hide_%s();" % conn_id)
- print(" }")
- print("}")
- print("function select_cb_sel_%s() {" % conn_id)
- print(" document.getElementById(\"cb_sel_%s\").checked = true;" % conn_id)
- print(" javascript:show_%s();" % conn_id)
- print("}")
- print("function deselect_cb_sel_%s() {" % conn_id)
- print(" document.getElementById(\"cb_sel_%s\").checked = false;" % conn_id)
- print(" javascript:hide_%s();" % conn_id)
- print("}")
- print("function toggle_cb_sel_%s() {" % conn_id)
- print(" if (document.getElementById(\"cb_sel_%s\").checked) {" % conn_id)
- print(" document.getElementById(\"cb_sel_%s\").checked = false;" % conn_id)
- print(" } else {")
- print(" document.getElementById(\"cb_sel_%s\").checked = true;" % conn_id)
- print(" }")
- print(" javascript:show_if_cb_sel_%s();" % conn_id)
- print("}")
-
- # Select/Deselect/Toggle All Connections functions
- print("function select_all() {")
- for conn_id, frames_ids in common.dict_iteritems(comn.conn_to_frame_map):
- print(" javascript:select_cb_sel_%s();" % conn_id)
- print("}")
- print("function deselect_all() {")
- for conn_id, frames_ids in common.dict_iteritems(comn.conn_to_frame_map):
- print(" javascript:deselect_cb_sel_%s();" % conn_id)
- print("}")
- print("function toggle_all() {")
- for conn_id, frames_ids in common.dict_iteritems(comn.conn_to_frame_map):
- print(" javascript:toggle_cb_sel_%s();" % conn_id)
- print("}")
-
- #
- print("</script>")
- print("</head>")
- print("<body>")
- #
-
- # Table of contents
- print(text.web_page_toc())
-
- # Report how much data was skipped if --no-data switch in effect
- if comn.args.skip_all_data:
- print("--skip-all-data switch is in effect. %d log lines skipped" % comn.data_skipped)
- print("<p><hr>")
-
- # file(s) included in this doc
- print("<a name=\"c_logfiles\"></a>")
- print("<h3>Log files</h3>")
- print("<table><tr><th>Log</th> <th>Container name</th> <th>Version</th> <th>Mode</th>"
- "<th>Instances</th> <th>Log file path</th></tr>")
- for i in range(comn.n_logs):
- rtrlist = comn.routers[i]
- if len(rtrlist) > 0:
- print("<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>" %
- (common.log_letter_of(i), rtrlist[0].container_name, rtrlist[0].version, rtrlist[0].mode,
- str(len(rtrlist)), os.path.abspath(comn.log_fns[i])))
- else:
- print("<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>" %
- (common.log_letter_of(i), text.nbsp(), text.nbsp(),
- str(len(rtrlist)), os.path.abspath(comn.log_fns[i])))
- print("</table>")
- print("<hr>")
-
- # reboot chronology
- print("<a name=\"c_rtrinstances\"></a>")
- print("<h3>Router Reboot Chronology</h3>")
- print("<table><tr><th>Log</th> <th>Time</th> <th>Container name</th> ")
- for i in range(len(comn.routers)):
- print("<td>%s</td>" % common.log_letter_of(i))
- print("</tr>")
- for rr in rr_tree:
- print("<tr><td>%s</td><td>%s</td><td>%s</td>" %
- (rr.router.iname, rr.datetime, rr.router.container_name))
- for i in range(len(comn.routers)):
- print("<td>%s</td> " % (rr.router.iname if i == rr.router.log_index else text.nbsp()))
- print("</tr>")
- print("</table>")
- print("<hr>")
-
- # print the connection peer tables
- #
- # +------+--------------------+-----+--------------------+-------+-------+----------+--------+
- # | View | Router | Dir | Peer | Log | N | Transfer | AMQP |
- # | +-----------+--------+ +--------+-----------+ lines | links | bytes | errors |
- # | | container | connid | | connid | container | | | | |
- # +------+-----------+--------+-----+--------+-----------+-------+-------+----------+--------+
-
- print("<a name=\"c_connections\"></a>")
- print("<h3>Connections</h3>")
-
- print("<p>")
- print("<button onclick=\"javascript:select_all()\">Select All</button>")
- print("<button onclick=\"javascript:deselect_all()\">Deselect All</button>")
- print("<button onclick=\"javascript:toggle_all()\">Toggle All</button>")
- print("</p>")
-
- print("<h3>Connections by ConnectionId</h3>")
- print(
- "<table><tr> <th rowspan=\"2\">View</th> <th colspan=\"2\">Router</th> <th rowspan=\"2\">Dir</th> <th colspan=\"2\">Peer</th> <th rowspan=\"2\">Log lines</th> "
- "<th rowspan=\"2\">N links</th><th rowspan=\"2\">Transfer bytes</th> <th rowspan=\"2\">AMQP errors</th> <th rowspan=\"2\">Open time</th> <th rowspan=\"2\">Close time</th></tr>")
- print("<tr> <th>container</th> <th>connid</th> <th>connid</th> <th>container</th></tr>")
-
- tConn = 0
- tLines = 0
- tBytes = 0
- tErrs = 0
- tLinks = 0
- for rtrlist in comn.routers:
- for rtr in rtrlist:
- rid = rtr.container_name
- for conn in rtr.conn_list:
- tConn += 1
- id = rtr.conn_id(conn) # this router's full connid 'A0_3'
- peer = rtr.conn_peer_display.get(id, "") # peer container id
- peerconnid = comn.conn_peers_connid.get(id, "")
- n_links = rtr.details.links_in_connection(id)
- tLinks += n_links
- errs = sum(1 for plf in rtr.conn_to_frame_map[id] if plf.data.amqp_error)
- tErrs += errs
- stime = rtr.conn_open_time.get(id, text.nbsp())
- if stime != text.nbsp():
- stime = stime.datetime
- etime = rtr.conn_close_time.get(id, text.nbsp())
- if etime != text.nbsp():
- etime = etime.datetime
- print("<tr>")
- print("<td> <input type=\"checkbox\" id=\"cb_sel_%s\" " % id)
- print("checked=\"true\" onclick=\"javascript:show_if_cb_sel_%s()\"> </td>" % (id))
- print("<td>%s</td><td><a href=\"#cd_%s\">%s</a></td><td>%s</td><td>%s</td><td>%s</td><td>%s</td>"
- "<td>%d</td><td>%s</td><td>%d</td><td>%s</td><td>%s</td></tr>" %
- (rid, id, id, rtr.conn_dir[id], peerconnid, peer, rtr.conn_log_lines[id], n_links,
- rtr.conn_xfer_bytes[id], errs, stime, etime))
- tLines += rtr.conn_log_lines[id]
- tBytes += rtr.conn_xfer_bytes[id]
- print(
- "<td>Total</td><td>%d</td><td> </td><td> </td><td> </td><td> </td><td>%d</td><td>%d</td><td>%d</td><td>%d</td></tr>" %
- (tConn, tLines, tLinks, tBytes, tErrs))
- print("</table>")
-
- print("<h3>Router Restart and Connection chronology</h3>")
-
- cl = []
- for rtrlist in comn.routers:
- for rtr in rtrlist:
- rid = rtr.container_name
- cl.append(common.RestartRec(rtr.iname, rtr, "restart", rtr.restart_rec.datetime))
- for conn in rtr.conn_list:
- id = rtr.conn_id(conn)
- if id in rtr.conn_open_time:
- cl.append(common.RestartRec(id, rtr, "open", rtr.conn_open_time[id].datetime))
- if id in rtr.conn_close_time:
- cl.append(common.RestartRec(id, rtr, "close", rtr.conn_close_time[id].datetime))
- cl = sorted(cl, key=lambda lfl: lfl.datetime)
-
- print("<table><tr> <th>Time</th> <th>Id</th> <th>Event</th> <th>container</th> <th>connid</th> "
- "<th>Dir</th> <th>connid</th> <th>container</th>")
- for i in range(len(comn.routers)):
- print("<td>%s</td>" % common.log_letter_of(i))
- print("</tr>")
- for c in cl:
- if c.event == "restart":
- rid = c.router.container_name
- print("<tr><td>%s</td> <td>%s</td> <td><span style=\"background-color:yellow\">%s</span></td><td>%s</td> "
- "<td>%s</td> <td>%s</td><td>%s</td> <td>%s</td>" %
- (c.datetime, c.id, c.event, rid, "", "", "", ""))
- for i in range(len(comn.routers)):
- print("<td>%s</td> " % (c.id if i == c.router.log_index else text.nbsp()))
- print("</tr>")
- else:
- rid = c.router.container_name
- cdir = c.router.conn_dir[c.id]
- peer = c.router.conn_peer_display.get(c.id, "") # peer container id
- peerconnid = comn.conn_peers_connid.get(c.id, "")
- print("<tr><td>%s</td> <td>%s</td> <td>%s</td><td>%s</td> <td>%s</td> <td>%s</td><td>%s</td> <td>%s</td>" %
- (c.datetime, c.id, c.event, rid, c.id, cdir, peerconnid, peer))
- for i in range(len(comn.routers)):
- print("<td>%s</td> " % (text.nbsp()))
- print("</tr>")
- print("</table>")
- print("<hr>")
-
- # connection details
- print("<a name=\"c_conndetails\"></a>")
- print("<h3>Connection Details</h3>")
- if not comn.args.skip_detail:
- for rtrlist in comn.routers:
- for rtr in rtrlist:
- rtr.details.show_html()
- else:
- print ("details suppressed<br>")
- print("<hr>")
-
- # noteworthy log lines: highlight errors and stuff
- print("<a name=\"c_noteworthy\"></a>")
- print("<h3>Noteworthy</h3>")
- n_errors = 0
- n_settled = 0
- n_more = 0
- n_resume = 0
- n_aborted = 0
- n_drain = 0
- for plf in tree:
- if plf.data.amqp_error:
- n_errors += 1
- if plf.data.transfer_settled:
- n_settled += 1
- if plf.data.transfer_more:
- n_more += 1
- if plf.data.transfer_resume:
- n_resume += 1
- if plf.data.transfer_aborted:
- n_aborted += 1
- if plf.data.flow_drain:
- n_drain += 1
- # amqp errors
- print("<a href=\"javascript:toggle_node('noteworthy_errors')\">%s%s</a> AMQP errors: %d<br>" %
- (text.lozenge(), text.nbsp(), n_errors))
- print(" <div width=\"100%%\"; "
- "style=\"display:none; font-weight: normal; margin-bottom: 2px; margin-left: 10px\" "
- "id=\"noteworthy_errors\">")
- for plf in tree:
- if plf.data.amqp_error:
- show_noteworthy_line(plf, comn)
- print("</div>")
- # transfers with settled=true
- print("<a href=\"javascript:toggle_node('noteworthy_settled')\">%s%s</a> Presettled transfers: %d<br>" %
- (text.lozenge(), text.nbsp(), n_settled))
- print(" <div width=\"100%%\"; "
- "style=\"display:none; font-weight: normal; margin-bottom: 2px; margin-left: 10px\" "
- "id=\"noteworthy_settled\">")
- for plf in tree:
- if plf.data.transfer_settled:
- show_noteworthy_line(plf, comn)
- print("</div>")
- # transfers with more=true
- print("<a href=\"javascript:toggle_node('noteworthy_more')\">%s%s</a> Partial transfers with 'more' set: %d<br>" %
- (text.lozenge(), text.nbsp(), n_more))
- print(" <div width=\"100%%\"; "
- "style=\"display:none; font-weight: normal; margin-bottom: 2px; margin-left: 10px\" "
- "id=\"noteworthy_more\">")
- for plf in tree:
- if plf.data.transfer_more:
- show_noteworthy_line(plf, comn)
- print("</div>")
- # transfers with resume=true, whatever that is
- print("<a href=\"javascript:toggle_node('noteworthy_resume')\">%s%s</a> Resumed transfers: %d<br>" %
- (text.lozenge(), text.nbsp(), n_resume))
- print(" <div width=\"100%%\"; "
- "style=\"display:none; font-weight: normal; margin-bottom: 2px; margin-left: 10px\" "
- "id=\"noteworthy_resume\">")
- for plf in tree:
- if plf.data.transfer_resume:
- show_noteworthy_line(plf, comn)
- print("</div>")
- # transfers with abort=true
- print("<a href=\"javascript:toggle_node('noteworthy_aborts')\">%s%s</a> Aborted transfers: %d<br>" %
- (text.lozenge(), text.nbsp(), n_aborted))
- print(" <div width=\"100%%\"; "
- "style=\"display:none; font-weight: normal; margin-bottom: 2px; margin-left: 10px\" "
- "id=\"noteworthy_aborts\">")
- for plf in tree:
- if plf.data.transfer_aborted:
- show_noteworthy_line(plf, comn)
- print("</div>")
- # flow with drain=true
- print("<a href=\"javascript:toggle_node('noteworthy_drain')\">%s%s</a> Flow with 'drain' set: %d<br>" %
- (text.lozenge(), text.nbsp(), n_drain))
- print(" <div width=\"100%%\"; "
- "style=\"display:none; font-weight: normal; margin-bottom: 2px; margin-left: 10px\" "
- "id=\"noteworthy_drain\">")
- for plf in tree:
- if plf.data.flow_drain:
- show_noteworthy_line(plf, comn)
- print("</div>")
- print("<hr>")
-
- # the proton log lines
- # log lines in f_A_116
- # log line details in f_A_116_d
- print("<a name=\"c_logdata\"></a>")
- print("<h3>Log data</h3>")
- for plf in tree:
- l_dict = plf.data.described_type.dict
- print("<div width=\"100%%\" style=\"display:block margin-bottom: 2px\" id=\"%s\">" % plf.fid)
- print("<a name=\"%s\"></a>" % plf.fid)
- detailname = plf.fid + "_d" # type: str
- loz = "<a href=\"javascript:toggle_node('%s')\">%s%s</a>" % (detailname, text.lozenge(), text.nbsp())
- rtr = plf.router
- rid = comn.router_display_names[rtr.log_index]
-
- peerconnid = "%s" % comn.conn_peers_connid.get(plf.data.conn_id, "")
- peer = rtr.conn_peer_display.get(plf.data.conn_id, "") # peer container id
- print(loz, plf.datetime, ("%s#%d" % (plf.prefixi, plf.lineno)), rid, ("[%s]" % plf.data.conn_id),
- plf.data.direction, ("[%s]" % peerconnid), peer,
- plf.data.web_show_str, plf.data.disposition_display, "<br>")
- print(" <div width=\"100%%\"; "
- "style=\"display:none; font-weight: normal; margin-bottom: 2px; margin-left: 10px\" "
- "id=\"%s\">" %
- detailname)
- for key in sorted(common.dict_iterkeys(l_dict)):
- val = l_dict[key]
- print("%s : %s <br>" % (key, cgi.escape(str(val))))
- if plf.data.name == "transfer":
- print("Header and annotations : %s <br>" % plf.data.transfer_hdr_annos)
- print("</div>")
- print("</div>")
- print("<hr>")
-
- # data traversing network
- print("<a name=\"c_messageprogress\"></a>")
- print("<h3>Message progress</h3>")
- if not comn.args.skip_msg_progress:
- for i in range(0, comn.shorteners.short_data_names.len()):
- sname = comn.shorteners.short_data_names.shortname(i)
- size = 0
- for plf in comn.shorteners.short_data_names.customers(sname):
- size = plf.data.transfer_size
- break
- print("<a name=\"%s\"></a> <h4>%s (%s)" % (sname, sname, size))
- print(" <span> <a href=\"javascript:toggle_node('%s')\"> %s</a>" % ("data_" + sname, text.lozenge()))
- print(" <div width=\"100%%\"; style=\"display:none; font-weight: normal; margin-bottom: 2px\" id=\"%s\">" %
- ("data_" + sname))
- print(" ", comn.shorteners.short_data_names.longname(i, True))
- print("</div> </span>")
- print("</h4>")
- print("<table>")
- print(
- "<tr><th>Src</th> <th>Time</th> <th>Router</th> <th>ConnId</th> <th>Dir</th> <th>ConnId</th> <th>Peer</th> "
- "<th>T delta</th> <th>T elapsed</th><th>Settlement</th><th>S elapsed</th></tr>")
- t0 = None
- tlast = None
- for plf in comn.shorteners.short_data_names.customers(sname):
- if t0 is None:
- t0 = plf.datetime
- tlast = plf.datetime
- delta = "0.000000"
- epsed = "0.000000"
- else:
- delta = time_offset(plf.datetime, tlast)
- epsed = time_offset(plf.datetime, t0)
- tlast = plf.datetime
- sepsed = ""
- if plf.data.final_disposition is not None:
- sepsed = time_offset(plf.data.final_disposition.datetime, t0)
- rid = plf.router.iname
- peerconnid = "%s" % comn.conn_peers_connid.get(plf.data.conn_id, "")
- peer = plf.router.conn_peer_display.get(plf.data.conn_id, "") # peer container id
- print("<tr><td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> "
- "<td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> </tr>" %
- (plf.adverbl_link_to(), plf.datetime, rid, plf.data.conn_id, plf.data.direction,
- peerconnid, peer, delta, epsed,
- plf.data.disposition_display, sepsed))
- print("</table>")
-
- print("<hr>")
-
- # link names traversing network
- print("<a name=\"c_linkprogress\"></a>")
- print("<h3>Link name propagation</h3>")
- for i in range(0, comn.shorteners.short_link_names.len()):
- if comn.shorteners.short_link_names.len() == 0:
- break
- sname = comn.shorteners.short_link_names.prefixname(i)
- print("<a name=\"%s\"></a> <h4>%s" % (sname, sname))
- print(" <span> <div width=\"100%%\"; style=\"display:block; font-weight: normal; margin-bottom: 2px\" >")
- print(comn.shorteners.short_link_names.longname(i, True))
- print("</div> </span>")
- print("</h4>")
- print("<table>")
- print("<tr><th>src</th> <th>Time</th> <th>Router</th> <th>ConnId</th> <th>Dir</th> <th>ConnId> <th>Peer</th> "
- "<th>T delta</th> <th>T elapsed</th></tr>")
- t0 = None
- tlast = None
- for plf in tree:
- if plf.data.name == "attach" and plf.data.link_short_name == sname:
- if t0 is None:
- t0 = plf.datetime
- delta = "0.000000"
- epsed = "0.000000"
- else:
- delta = time_offset(plf.datetime, tlast)
- epsed = time_offset(plf.datetime, t0)
- tlast = plf.datetime
- rid = plf.router.iname
- peerconnid = "%s" % comn.conn_peers_connid.get(plf.data.conn_id, "")
- peer = plf.router.conn_peer_display.get(plf.data.conn_id, "") # peer container id
- print("<tr><td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> <td>%s</td> "
- "<td>%s</td> <td>%s</td> <td>%s</td></tr>" %
- (plf.adverbl_link_to(), plf.datetime, rid, plf.data.conn_id, plf.data.direction, peerconnid, peer,
- delta, epsed))
- print("</table>")
-
- print("<hr>")
-
- # short data index
- print("<a name=\"c_rtrdump\"></a>")
- comn.shorteners.short_rtr_names.htmlDump(False)
- print("<hr>")
-
- print("<a name=\"c_peerdump\"></a>")
- comn.shorteners.short_peer_names.htmlDump(False)
- print("<hr>")
-
- print("<a name=\"c_linkdump\"></a>")
- comn.shorteners.short_link_names.htmlDump(True)
- print("<hr>")
-
- print("<a name=\"c_msgdump\"></a>")
- comn.shorteners.short_data_names.htmlDump(True)
- print("<hr>")
-
- # link state info
- # merge link state and restart records into single time based list
- cl = []
- for rtrlist in comn.routers:
- for rtr in rtrlist:
- rid = rtr.container_name
- cl.append(common.RestartRec(rtr.iname, rtr, "restart", rtr.restart_rec.datetime))
- for plf in ls_tree:
- if "costs" in plf.line:
- cl.append(common.RestartRec("ls", plf, "ls", plf.datetime))
- cl = sorted(cl, key=lambda lfl: lfl.datetime)
-
- # create a map of lists for each router
- # the list holds the name of other routers for which the router publishes a cost
- costs_pub = {}
- for i in range(0, comn.n_logs):
- costs_pub[comn.router_ids[i]] = []
-
- # cur_costs is a 2D array of costs used to tell when cost calcs have stabilized
- # Each incoming LS cost line replaces a row in this table
- # cur_costs tracks only interior routers
- interior_rtrs = []
- for rtrs in comn.routers:
- if rtrs[0].is_interior():
- interior_rtrs.append(rtrs[0].container_name)
-
- PEER_COST_REBOOT = -1
- PEER_COST_ABSENT = 0
- def new_costs_row(val):
- """
- return a costs row.
- :param val: -1 when router reboots, 0 when router log line processed
- :return:
- """
- res = {}
- for rtr in interior_rtrs:
- res[rtr] = val
- return res
-
- cur_costs = {}
- for rtr in interior_rtrs:
- cur_costs[rtr] = new_costs_row(PEER_COST_REBOOT)
-
- print("<a name=\"c_ls\"></a>")
- print("<h3>Routing link state</h3>")
- print("<h4>Link state costs</h4>")
- print("<table>")
- print("<tr><th>Time</th> <th>Router</th>")
- for i in range(0, comn.n_logs):
- print("<th>%s</th>" % common.log_letter_of(i))
- print("</tr>")
- for c in cl:
- if c.event == "ls":
- # link state computed costs and router reachability
- plf = c.router # cruel overload here: router is a parsed line not a router
- # Processing: Computed costs: {u'A': 1, u'C': 51L, u'B': 101L}
- print("<tr><td>%s</td> <td>%s</td>" % (plf.datetime, ("%s#%d" % (plf.router.iname, plf.lineno))))
- try:
- line = plf.line
- sti = line.find("{")
- line = line[sti:]
- l_dict = ast.literal_eval(line)
- costs_row = new_costs_row(PEER_COST_ABSENT)
- for i in range(0, comn.n_logs):
- if len(comn.routers[i]) > 0:
- tst_name = comn.routers[i][0].container_name
- if tst_name in l_dict:
- val = l_dict[tst_name]
- costs_row[tst_name] = val
- elif i == plf.router.log_index:
- val = text.nbsp()
- else:
- val = "<span style=\"background-color:orange\">%s</span>" % (text.nbsp() * 2)
- else:
- val = "<span style=\"background-color:orange\">%s</span>" % (text.nbsp() * 2)
- print("<td>%s</td>" % val)
- # track costs published when there is no column to put the number
- tgts = costs_pub[c.router.router.container_name]
- for k, v in common.dict_iteritems(l_dict):
- if k not in comn.router_ids:
- if k not in tgts:
- tgts.append(k) # this cost went unreported
- # update this router's cost view in running table
- if plf.router.is_interior():
- cur_costs[plf.router.container_name] = costs_row
- except:
- pass
- print("</tr>")
- # if the costs are stable across all routers then put an indicator in table
- costs_stable = True
- for c_rtr in interior_rtrs:
- for r_rtr in interior_rtrs:
- if r_rtr != c_rtr \
- and (cur_costs[r_rtr][c_rtr] != cur_costs[c_rtr][r_rtr] \
- or cur_costs[c_rtr][r_rtr] <= PEER_COST_ABSENT):
- costs_stable = False
- break
- if not costs_stable:
- break
- if costs_stable:
- print("<tr><td><span style=\"background-color:green\">stable</span></td></tr>")
- else:
- # restart
- print("<tr><td>%s</td> <td>%s</td>" % (c.datetime, ("%s restart" % (c.router.iname))))
- for i in range(0, comn.n_logs):
- color = "green" if i == c.router.log_index else "orange"
- print("<td><span style=\"background-color:%s\">%s</span></td>" % (color, text.nbsp() * 2))
- print("</tr>")
- if c.router.is_interior():
- cur_costs[c.router.container_name] = new_costs_row(PEER_COST_REBOOT)
- print("</table>")
- print("<br>")
-
- # maybe display cost declarations that were not displayed
- costs_clean = True
- for k, v in common.dict_iteritems(costs_pub):
- if len(v) > 0:
- costs_clean = False
- break
- if not costs_clean:
- print("<h4>Router costs declared in logs but not displayed in Link state cost table</h4>")
- print("<table>")
- print("<tr><th>Router</th><Peers whose logs are absent</th></tr>")
- for k, v in common.dict_iteritems(costs_pub):
- if len(v) > 0:
- print("<tr><td>%s</td><td>%s</td></tr>" % (k, str(v)))
- print("</table>")
- print("<br>")
-
- print("<a href=\"javascript:toggle_node('ls_costs')\">%s%s</a> Link state costs data<br>" %
- (text.lozenge(), text.nbsp()))
- print(" <div width=\"100%%\"; "
- "style=\"display:none; font-weight: normal; margin-bottom: 2px; margin-left: 10px\" "
- "id=\"ls_costs\">")
- print("<table>")
- print("<tr><th>Time</th> <th>Router</th> <th>Name</th> <th>Log</th></tr>")
- for plf in ls_tree:
- if "costs" in plf.line:
- print("<tr><td>%s</td> <td>%s</td>" % (plf.datetime, ("%s#%d" % (plf.router.iname, plf.lineno))))
- print("<td>%s</td>" % plf.router.container_name)
- print("<td>%s</td></tr>" % plf.line)
- print("</table>")
- print("</div>")
-
- print("<hr>")
-
- print("</body>")
-
-
-def main(argv):
- try:
- main_except(argv)
- return 0
- except Exception as e:
- traceback.print_exc()
- return 1
-
-
-if __name__ == "__main__":
- sys.exit(main(sys.argv))
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/b7ab3390/bin/log_scraper/test_data.py
----------------------------------------------------------------------
diff --git a/bin/log_scraper/test_data.py b/bin/log_scraper/test_data.py
deleted file mode 100755
index a711931..0000000
--- a/bin/log_scraper/test_data.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-from __future__ import unicode_literals
-from __future__ import division
-from __future__ import absolute_import
-from __future__ import print_function
-
-import sys
-import traceback
-
-class TestData():
- '''
- Extract list of test log lines from a data file.
- The file holds literal log lines from some noteworthy test logs.
- Embedding the lines as a python source code data statement involves escaping
- double quotes and runs the risk of corrupting the data.
- '''
- def __init__(self, fn="test_data/test_data.txt"):
- with open(fn, 'r') as f:
- self.lines = [line.rstrip('\n') for line in f]
-
- def data(self):
- return self.lines
-
-
-if __name__ == "__main__":
-
- try:
- datasource = TestData()
- for line in datasource.data():
- print (line)
- pass
- except:
- traceback.print_exc(file=sys.stdout)
- pass
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/b7ab3390/bin/log_scraper/test_data/A-two-instances.log
----------------------------------------------------------------------
diff --git a/bin/log_scraper/test_data/A-two-instances.log b/bin/log_scraper/test_data/A-two-instances.log
deleted file mode 100644
index 05d43f3..0000000
--- a/bin/log_scraper/test_data/A-two-instances.log
+++ /dev/null
@@ -1,113 +0,0 @@
-2018-10-15 10:57:32.149418 -0400 AGENT (debug) Add entity: LogEntity(enable=trace+, identity=log/DEFAULT, includeSource=False, includeTimestamp=True, module=DEFAULT, name=log/DEFAULT, outputFile=taj-GRN.log, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.149739 -0400 AGENT (debug) Add entity: LogEntity(identity=log/HTTP, module=HTTP, name=log/HTTP, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.149990 -0400 AGENT (debug) Add entity: LogEntity(identity=log/ROUTER_LS, module=ROUTER_LS, name=log/ROUTER_LS, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.150202 -0400 AGENT (debug) Add entity: LogEntity(identity=log/PYTHON, module=PYTHON, name=log/PYTHON, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.150459 -0400 AGENT (debug) Add entity: LogEntity(identity=log/ROUTER_MA, module=ROUTER_MA, name=log/ROUTER_MA, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.150681 -0400 AGENT (debug) Add entity: LogEntity(identity=log/CONN_MGR, module=CONN_MGR, name=log/CONN_MGR, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.150949 -0400 AGENT (debug) Add entity: LogEntity(identity=log/ROUTER_HELLO, module=ROUTER_HELLO, name=log/ROUTER_HELLO, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.151172 -0400 AGENT (debug) Add entity: LogEntity(identity=log/SERVER, module=SERVER, name=log/SERVER, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.151411 -0400 AGENT (debug) Add entity: LogEntity(identity=log/POLICY, module=POLICY, name=log/POLICY, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.151673 -0400 AGENT (debug) Add entity: LogEntity(identity=log/CONTAINER, module=CONTAINER, name=log/CONTAINER, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.151943 -0400 AGENT (debug) Add entity: LogEntity(identity=log/AGENT, module=AGENT, name=log/AGENT, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.152198 -0400 AGENT (debug) Add entity: LogEntity(identity=log/ERROR, module=ERROR, name=log/ERROR, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.152471 -0400 AGENT (debug) Add entity: LogEntity(identity=log/ROUTER_CORE, module=ROUTER_CORE, name=log/ROUTER_CORE, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.152716 -0400 AGENT (debug) Add entity: LogEntity(identity=log/ROUTER, module=ROUTER, name=log/ROUTER, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.152958 -0400 AGENT (debug) Add entity: LogEntity(identity=log/AUTHSERVICE, module=AUTHSERVICE, name=log/AUTHSERVICE, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:57:32.336561 -0400 AGENT (debug) Add entity: RouterEntity(allowResumableLinkRoute=True, allowUnsettledMulticast=False, area=0, defaultDistribution=balanced, helloIntervalSeconds=1, helloMaxAgeSeconds=3, hostName=taj.localdomain, id=central-qdr-green, mode=interior, raIntervalFluxSeconds=4, raIntervalSeconds=30, remoteLsMaxAgeSeconds=60, saslConfigName=qdrouterd, type=org.apache.qpid.dispatch.router, workerThreads=4)
-2018-10-15 10:57:32.337966 -0400 SERVER (warning) HTTP support is not available
-2018-10-15 10:57:32.338005 -0400 SERVER (info) Container Name: central-qdr-green
-2018-10-15 10:57:32.338082 -0400 CONTAINER (trace) Container Initialized
-2018-10-15 10:57:32.338183 -0400 CONTAINER (trace) Node Type Registered - router
-2018-10-15 10:57:32.338237 -0400 CONTAINER (trace) Node of type 'router' installed as default node
-2018-10-15 10:57:32.338297 -0400 ROUTER (info) Router started in Interior mode, area=0 id=central-qdr-green
-2018-10-15 10:57:32.338313 -0400 ROUTER (info) Version: 1.5.0-SNAPSHOT
-2018-10-15 10:57:32.338374 -0400 POLICY (trace) Policy Initialized
-2018-10-15 10:57:32.339516 -0400 ROUTER (info) Router Engine Instantiated: id=central-qdr-green instance=1539615452 max_routers=128
-2018-10-15 10:57:32.378087 -0400 SERVER (notice) Listening on 0.0.0.0:amqp
-2018-10-15 10:57:32.378189 -0400 SERVER (notice) Listening on 0.0.0.0:20001
-2018-10-15 10:57:33.342104 -0400 ROUTER_CORE (trace) Core action 'send_to'
-2018-10-15 10:57:33.342320 -0400 ROUTER_HELLO (trace) SENT: HELLO(id=central-qdr-green pv=1 area=0 inst=1539615452 seen=[])
-2018-10-15 10:57:42.865701 -0400 SERVER (trace) [1]: Accepting incoming connection to '0.0.0.0:amqp'
-2018-10-15 10:57:42.865785 -0400 POLICY (trace) ALLOW Connection '127.0.0.1' based on global connection count. nConnections= 1
-2018-10-15 10:57:42.865801 -0400 SERVER (info) [1]: Accepted connection to 0.0.0.0:amqp from 127.0.0.1:55854
-2018-10-15 10:57:42.865844 -0400 SERVER (trace) [1]: <- AMQP
-2018-10-15 10:57:42.865872 -0400 SERVER (trace) [1]:0 <- @open(16) [container-id="d45d5f6a-ade4-437b-92eb-5978ddbffb4e", hostname="127.0.0.1", channel-max=32767]
-2018-10-15 10:57:42.865889 -0400 SERVER (trace) [1]:0 <- @begin(17) [next-outgoing-id=0, incoming-window=2147483647, outgoing-window=2147483647]
-2018-10-15 10:57:42.865926 -0400 SERVER (trace) [1]:0 <- @attach(18) [name="29b38f24-8c87-4964-8a19-0b4f3aa44660", handle=0, role=true, snd-settle-mode=2, rcv-settle-mode=0, source=@source(40) [address="collectd/telemetry", durable=0, timeout=0, dynamic=false], target=@target(41) [durable=0, timeout=0, dynamic=false], initial-delivery-count=0, max-message-size=0]
-2018-10-15 10:57:42.865981 -0400 SERVER (trace) [1]: -> AMQP
-2018-10-15 10:57:42.866110 -0400 ROUTER_CORE (trace) Core action 'connection_opened'
-2018-10-15 10:57:42.866198 -0400 SERVER (trace) [1]:0 -> @open(16) [container-id="central-qdr-green", max-frame-size=16384, channel-max=32767, idle-time-out=8000, offered-capabilities=:"ANONYMOUS-RELAY", properties={:product="qpid-dispatch-router", :version="1.5.0-SNAPSHOT", :"qd.conn-id"=1}]
-2018-10-15 10:57:42.866217 -0400 ROUTER_CORE (trace) Core action 'link_first_attach'
-2018-10-15 10:57:42.866263 -0400 DEFAULT (trace) Parse tree search for 'collectd/telemetry'
-2018-10-15 10:57:42.866274 -0400 DEFAULT (trace) Parse tree match not found
-2018-10-15 10:57:42.866284 -0400 DEFAULT (trace) Parse tree search for 'collectd/telemetry'
-2018-10-15 10:57:42.866323 -0400 SERVER (trace) [1]:0 -> @begin(17) [remote-channel=0, next-outgoing-id=0, incoming-window=2147483647, outgoing-window=2147483647]
-2018-10-15 10:57:42.866441 -0400 SERVER (trace) [1]:0 -> @attach(18) [name="29b38f24-8c87-4964-8a19-0b4f3aa44660", handle=0, role=false, snd-settle-mode=2, rcv-settle-mode=0, source=@source(40) [address="collectd/telemetry", durable=0, expiry-policy=:"session-end", timeout=0, dynamic=false], target=@target(41) [durable=0, expiry-policy=:"session-end", timeout=0, dynamic=false], initial-delivery-count=0, max-message-size=0]
-2018-10-15 10:57:42.866591 -0400 SERVER (trace) [1]:0 <- @flow(19) [next-incoming-id=0, incoming-window=2147483647, next-outgoing-id=0, outgoing-window=2147483647, handle=0, delivery-count=0, link-credit=10, drain=false]
-2018-10-15 10:59:07.454660 -0400 AGENT (debug) Add entity: LogEntity(enable=trace+, identity=log/DEFAULT, includeSource=False, includeTimestamp=True, module=DEFAULT, name=log/DEFAULT, outputFile=taj-GRN.log, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.454984 -0400 AGENT (debug) Add entity: LogEntity(identity=log/HTTP, module=HTTP, name=log/HTTP, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.455250 -0400 AGENT (debug) Add entity: LogEntity(identity=log/ROUTER_LS, module=ROUTER_LS, name=log/ROUTER_LS, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.455460 -0400 AGENT (debug) Add entity: LogEntity(identity=log/PYTHON, module=PYTHON, name=log/PYTHON, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.455720 -0400 AGENT (debug) Add entity: LogEntity(identity=log/ROUTER_MA, module=ROUTER_MA, name=log/ROUTER_MA, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.455943 -0400 AGENT (debug) Add entity: LogEntity(identity=log/CONN_MGR, module=CONN_MGR, name=log/CONN_MGR, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.456202 -0400 AGENT (debug) Add entity: LogEntity(identity=log/ROUTER_HELLO, module=ROUTER_HELLO, name=log/ROUTER_HELLO, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.456422 -0400 AGENT (debug) Add entity: LogEntity(identity=log/SERVER, module=SERVER, name=log/SERVER, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.456658 -0400 AGENT (debug) Add entity: LogEntity(identity=log/POLICY, module=POLICY, name=log/POLICY, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.456926 -0400 AGENT (debug) Add entity: LogEntity(identity=log/CONTAINER, module=CONTAINER, name=log/CONTAINER, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.457184 -0400 AGENT (debug) Add entity: LogEntity(identity=log/AGENT, module=AGENT, name=log/AGENT, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.457435 -0400 AGENT (debug) Add entity: LogEntity(identity=log/ERROR, module=ERROR, name=log/ERROR, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.457704 -0400 AGENT (debug) Add entity: LogEntity(identity=log/ROUTER_CORE, module=ROUTER_CORE, name=log/ROUTER_CORE, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.457951 -0400 AGENT (debug) Add entity: LogEntity(identity=log/ROUTER, module=ROUTER, name=log/ROUTER, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.458187 -0400 AGENT (debug) Add entity: LogEntity(identity=log/AUTHSERVICE, module=AUTHSERVICE, name=log/AUTHSERVICE, type=org.apache.qpid.dispatch.log)
-2018-10-15 10:59:07.583032 -0400 AGENT (debug) Add entity: RouterEntity(allowResumableLinkRoute=True, allowUnsettledMulticast=False, area=0, defaultDistribution=balanced, helloIntervalSeconds=1, helloMaxAgeSeconds=3, hostName=taj.localdomain, id=central-qdr-green, mode=interior, raIntervalFluxSeconds=4, raIntervalSeconds=30, remoteLsMaxAgeSeconds=60, saslConfigName=qdrouterd, type=org.apache.qpid.dispatch.router, workerThreads=4)
-2018-10-15 10:59:07.584383 -0400 SERVER (warning) HTTP support is not available
-2018-10-15 10:59:07.584421 -0400 SERVER (info) Container Name: central-qdr-green
-2018-10-15 10:59:07.584498 -0400 CONTAINER (trace) Container Initialized
-2018-10-15 10:59:07.584600 -0400 CONTAINER (trace) Node Type Registered - router
-2018-10-15 10:59:07.584656 -0400 CONTAINER (trace) Node of type 'router' installed as default node
-2018-10-15 10:59:07.584730 -0400 ROUTER (info) Router started in Interior mode, area=0 id=central-qdr-green
-2018-10-15 10:59:07.584749 -0400 ROUTER (info) Version: 1.5.0-SNAPSHOT
-2018-10-15 10:59:07.584807 -0400 POLICY (trace) Policy Initialized
-2018-10-15 10:59:07.586002 -0400 ROUTER (info) Router Engine Instantiated: id=central-qdr-green instance=1539615547 max_routers=128
-2018-10-15 10:59:07.586126 -0400 ROUTER_CORE (info) Initializing core module: edge_router
-2018-10-15 10:59:07.586184 -0400 ROUTER_CORE (info) Initializing core module: core_test_hooks
-2018-10-15 10:59:07.622921 -0400 SERVER (notice) Operational, 4 Threads Running (process ID 22085)
-2018-10-15 10:59:07.622943 -0400 SERVER (info) Running in DEBUG Mode
-2018-10-15 10:59:07.623098 -0400 SERVER (notice) Listening on 0.0.0.0:20001
-2018-10-15 10:59:07.623218 -0400 SERVER (notice) Listening on 0.0.0.0:amqp
-2018-10-15 10:59:08.587951 -0400 ROUTER_CORE (trace) Core action 'send_to'
-2018-10-15 10:59:08.588089 -0400 ROUTER_HELLO (trace) SENT: HELLO(id=central-qdr-green pv=1 area=0 inst=1539615547 seen=[])
-2018-10-15 10:59:08.588233 -0400 ROUTER_CORE (trace) Core action 'send_to'
-2018-10-15 10:59:08.588319 -0400 ROUTER_LS (trace) SENT: RA(id=central-qdr-green pv=1 area=0 inst=1539615547 ls_seq=0 mobile_seq=0)
-2018-10-15 10:59:08.588402 -0400 ROUTER_CORE (trace) Core action 'process_tick'
-2018-10-15 10:59:09.588673 -0400 ROUTER_CORE (trace) Core action 'send_to'
-2018-10-15 10:59:09.588779 -0400 ROUTER_HELLO (trace) SENT: HELLO(id=central-qdr-green pv=1 area=0 inst=1539615547 seen=[])
-2018-10-15 10:59:09.590682 -0400 ROUTER_CORE (trace) Core action 'process_tick'
-2018-10-15 10:59:09.621170 -0400 SERVER (trace) Accepting connection on 0.0.0.0:20001
-2018-10-15 10:59:09.621347 -0400 SERVER (trace) [1]: Accepting incoming connection to '0.0.0.0:20001'
-2018-10-15 10:59:09.621503 -0400 POLICY (trace) ALLOW Connection '192.168.1.7' based on global connection count. nConnections= 1
-2018-10-15 10:59:09.621530 -0400 SERVER (info) [1]: Accepted connection to 0.0.0.0:20001 from 192.168.1.7:36494
-2018-10-15 10:59:09.621569 -0400 SERVER (trace) [1]: <- SASL
-2018-10-15 10:59:09.621595 -0400 SERVER (trace) [1]: -> SASL
-2018-10-15 10:59:09.622163 -0400 SERVER (trace) Accepting connection on 0.0.0.0:20001
-2018-10-15 10:59:09.622280 -0400 SERVER (trace) [2]: Accepting incoming connection to '0.0.0.0:20001'
-2018-10-15 10:59:09.622387 -0400 POLICY (trace) ALLOW Connection '192.168.1.7' based on global connection count. nConnections= 2
-2018-10-15 10:59:09.622410 -0400 SERVER (info) [2]: Accepted connection to 0.0.0.0:20001 from 192.168.1.7:36496
-2018-10-15 10:59:09.624269 -0400 SERVER (trace) [2]: <- SASL
-2018-10-15 10:59:09.624310 -0400 SERVER (trace) [2]: -> SASL
-2018-10-15 10:59:09.624629 -0400 SERVER (trace) [1]:0 -> @sasl-mechanisms(64) [sasl-server-mechanisms=@PN_SYMBOL[:ANONYMOUS, :"DIGEST-MD5", :PLAIN]]
-2018-10-15 10:59:09.624660 -0400 SERVER (trace) [2]:0 -> @sasl-mechanisms(64) [sasl-server-mechanisms=@PN_SYMBOL[:ANONYMOUS, :"DIGEST-MD5", :PLAIN]]
-2018-10-15 10:59:09.626696 -0400 SERVER (trace) [1]:0 <- @sasl-init(65) [mechanism=:ANONYMOUS, initial-response=b"anonymous@ratchet.localdomain"]
-2018-10-15 10:59:09.626787 -0400 SERVER (trace) [2]:0 <- @sasl-init(65) [mechanism=:ANONYMOUS, initial-response=b"anonymous@ratchet.localdomain"]
-2018-10-15 10:59:09.626861 -0400 SERVER (trace) [2]:0 -> @sasl-outcome(68) [code=0]
-2018-10-15 10:59:09.626894 -0400 SERVER (trace) [1]:0 -> @sasl-outcome(68) [code=0]
-2018-10-15 10:59:09.630886 -0400 SERVER (trace) [1]: <- AMQP
-2018-10-15 10:59:09.630971 -0400 SERVER (trace) [1]:0 <- @open(16) [container-id="collectd", hostname="taj", max-frame-size=16384, channel-max=32767, idle-time-out=8000, offered-capabilities=:"ANONYMOUS-RELAY", properties={:product="qpid-dispatch-router", :version="1.4.0-SNAPSHOT", :"qd.conn-id"=5}]
-2018-10-15 10:59:09.631167 -0400 SERVER (trace) [1]: -> AMQP
-2018-10-15 10:59:09.631197 -0400 ROUTER_CORE (trace) Core action 'connection_opened'
-2018-10-15 10:59:09.631246 -0400 SERVER (trace) [1]:0 -> @open(16) [container-id="central-qdr-green", max-frame-size=16384, channel-max=32767, idle-time-out=8000, offered-capabilities=:"ANONYMOUS-RELAY", properties={:product="qpid-dispatch-router", :version="1.5.0-SNAPSHOT", :"qd.conn-id"=1}]
-2018-10-15 10:59:09.631949 -0400 SERVER (trace) [2]: <- AMQP
-2018-10-15 10:59:09.632028 -0400 SERVER (trace) [2]:0 <- @open(16) [container-id="central-qdr-blue", hostname="taj", max-frame-size=16384, channel-max=32767, idle-time-out=8000, offered-capabilities=:"ANONYMOUS-RELAY", properties={:product="qpid-dispatch-router", :version="1.4.0-SNAPSHOT", :"qd.conn-id"=5}]
-2018-10-15 10:59:09.632117 -0400 ROUTER_CORE (trace) Core action 'connection_opened'
-2018-10-15 10:59:09.632148 -0400 SERVER (trace) [2]: -> AMQP
-2018-10-15 10:59:09.632221 -0400 SERVER (trace) [2]:0 -> @open(16) [container-id="central-qdr-green", max-frame-size=16384, channel-max=32767, idle-time-out=8000, offered-capabilities=:"ANONYMOUS-RELAY", properties={:product="qpid-dispatch-router", :version="1.5.0-SNAPSHOT", :"qd.conn-id"=2}]
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/b7ab3390/bin/log_scraper/test_data/test_data.txt
----------------------------------------------------------------------
diff --git a/bin/log_scraper/test_data/test_data.txt b/bin/log_scraper/test_data/test_data.txt
deleted file mode 100644
index f659a34..0000000
--- a/bin/log_scraper/test_data/test_data.txt
+++ /dev/null
@@ -1,30 +0,0 @@
-2018-11-21 15:47:09.727570 -0500 SERVER (trace) [7]:1 <- @attach(18) [name="qpid-jms:sender:ID:23d7f58d-9bb1-4a8a-9701-d6eb7f7ec15e:1:1:1:test.Q0", handle=0, role=false, snd-settle-mode=0, rcv-settle-mode=0, source=@source(40) [address="ID:23d7f58d-9cc1-4a8a-9701-d6eb7f7ec15e:1:1:1", durable=0, expiry-policy=:"session-end", timeout=0, dynamic=false, outcomes=@PN_SYMBOL[:"amqp:accepted:list", :"amqp:rejected:list", :"amqp:released:list", :"amqp:modified:list"]], target=@target(41) [address="test.Q0", durable=0, expiry-policy=:"session-end", timeout=0, dynamic=false, capabilities=@PN_SYMBOL[:queue]], incomplete-unsettled=false, initial-delivery-count=0, desired-capabilities=@PN_SYMBOL[:"DELAYED_DELIVERY"]]
-2018-11-18 10:52:52.34008 -0500 SERVER (trace) [255]:2 <- @attach(18) [name="qpid-jms:receiver:ID:c50ab67b-0ff1-41fe-84a6-7a7bace101ec:16263:2:1:some-queue", handle=0, role=true, snd-settle-mode=0, rcv-settle-mode=0, source=@source(40) [address="some-queue", durable=0, expiry-policy=:"link-detach", timeout=0, dynamic=false, default-outcome=@modified(39) [delivery-failed=true], outcomes=@PN_SYMBOL[:"amqp:accepted:list", :"amqp:rejected:list", :"amqp:released:list", :"amqp:modified:list"], capabilities=@PN_SYMBOL[:queue]], target=@target(41) []]
-2018-07-20 10:58:40.176528 -0400 SERVER (trace) [2] Connecting to 127.0.0.1:23731 (/home/chug/git/qpid-dispatch/src/server.c:1052)
-2018-07-20 10:58:40.176628 -0400 SERVER (trace) [2]: -> SASL (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:40.176841 -0400 SERVER (trace) [2]: <- SASL (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:40.176869 -0400 SERVER (trace) [2]:0 <- @sasl-mechanisms(64) [sasl-server-mechanisms=@PN_SYMBOL[:ANONYMOUS]] (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:40.178334 -0400 SERVER (trace) [2]:0 -> @sasl-init(65) [mechanism=:ANONYMOUS, initial-response=b"anonymous@ratchet.localdomain"] (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:40.178470 -0400 SERVER (trace) [2]:0 <- @sasl-outcome(68) [code=0] (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:40.178756 -0400 SERVER (trace) [2]: <- AMQP (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:40.178799 -0400 SERVER (trace) [2]:0 <- @open(16) [container-id="A", max-frame-size=16384, channel-max=32767, idle-time-out=60000, offered-capabilities=:"ANONYMOUS-RELAY", properties={:product="qpid-dispatch-router", :version="1.3.0-SNAPSHOT"}] (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:40.179187 -0400 SERVER (trace) [2]:0 -> @begin(17) [next-outgoing-id=0, incoming-window=2147483647, outgoing-window=2147483647] (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:40.180136 -0400 SERVER (trace) [2]:0 <- @attach(18) [name="qdlink.xgqQHT+EqTH14nA", handle=3, role=true, snd-settle-mode=2, rcv-settle-mode=0, source=@source(40) [durable=0, expiry-policy=:"session-end", timeout=0, dynamic=false, capabilities=:"qd.router-data"], target=@target(41) [durable=0, expiry-policy=:"session-end", timeout=0, dynamic=false, capabilities=:"qd.router-data"], initial-delivery-count=0, max-message-size=0] (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:40.180171 -0400 SERVER (trace) [2]:0 <- @flow(19) [next-incoming-id=0, incoming-window=2147483647, next-outgoing-id=0, outgoing-window=2147483647, handle=1, delivery-count=0, link-credit=1000, drain=false] (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:41.811112 -0400 SERVER (trace) [2]:0 <- @transfer(20) [handle=2, delivery-id=1, delivery-tag=b"\x0f\x00\x00\x00\x00\x00\x00\x00", message-format=0] (320) "\x00SpE\x00Sr\xd1\x00\x00\x00k\x00\x00\x00\x08\xa3\x0bx-opt-qd.to\xa1\x15_topo/0/C/$management\xa3\x0ex-opt-qd.trace\xd0\x00\x00\x00\x09\x00\x00\x00\x01\xa1\x030/A\xa3\x10x-opt-qd.ingress\xa1\x030/A\xa3\x09x-opt-qd.\xa1\x01X\x00Ss\xd0\x00\x00\x002\x00\x00\x00\x06@@@@\xa1$amqp:/_topo/0/A/temp.Q+EzDvtPUeSkHaD\xa0\x0212\x00St\xd1\x00\x00\x00j\x00\x00\x00\x08\xa1\x09operation\xa1\x05QUERY\xa1\x0aentityType\xa1\x1forg.apache.qpid.dispatch.router\xa1\x04type\xa1\x13org.amqp.management\xa1\x04name\xa1\x04self\x00Sw\xd1\x00\x00\x00\x15\x00\x00\x00\x02\xa1\x0eattributeNamesE" (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:41.811312 -0400 SERVER (trace) [2]:0 -> @disposition(21) [role=true, first=1, settled=true, state=@accepted(36) []] (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:55.893356 -0400 SERVER (trace) [2]:0 -> @close(24) [error=@error(29) [condition=:"amqp:connection:framing-error", description="connection aborted"]] (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:55.893366 -0400 SERVER (trace) [2]: <- EOS (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:56.180136 -0400 SERVER (trace) [2]:0 <- @attach(18) [name="97e94a05-1e69-492f-b9b4-282beb79484e#_aafc09ac-efc1-4f51-b038-c529408b7b61", handle=0, role=true, snd-settle-mode=2, rcv-settle-mode=0, source=@source(40) [durable=0, timeout=0, dynamic=true, dynamic-node-properties={:"lifetime-policy"=@:"amqp:delete-on-close:list" []}], target=@target(41) [durable=0, timeout=0, dynamic=false], initial-delivery-count=0, max-message-size=0] (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:57.368594 -0400 SERVER (trace) [2]:0 <- @transfer(20) [handle=2, delivery-id=160, delivery-tag=b""\x03\x00\x00\x00\x00\x00\x00", message-format=0] (120) "\x00SpE\x00Sr\xd1\x00\x00\x00`\x00\x00\x00\x08\xa3\x0bx-opt-qd.to\xa1\x0aclosest/02\xa3\x0ex-opt-qd.trace\xd0\x00\x00\x00\x09\x00\x00\x00\x01\xa1\x030/A\xa3\x10x-opt-qd.ingress\xa1\x030/A\xa3\x09x-opt-qd.\xa1\x01X\x00SsE\x00Swq\x00\x00\x023" (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-07-20 10:58:57.468234 -0400 SERVER (trace) [2]:0 <- @transfer(20) [handle=2, delivery-id=129, delivery-tag=b"\xff\x02\x00\x00\x00\x00\x00\x00", message-format=0] (120) "\x00SpE\x00Sr\xd1\x00\x00\x00`\x00\x00\x00\x08\xa3\x0bx-opt-qd.to\xa1\x0aclosest/02\xa3\x0ex-opt-qd.trace\xd0\x00\x00\x00\x09\x00\x00\x00\x01\xa1\x030/A\xa3\x10x-opt-qd.ingress\xa1\x030/A\xa3\x09x-opt-qd.\xa1\x01X\x00SsE\x00Swq\x00\x00\x02"" (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-08-03 10:59:37.457537 -0400 SERVER (trace) [5]:0 -> @transfer(20) [handle=0, delivery-id=6, delivery-tag=b" \x00\x00\x00\x00\x00\x00\x00", message-format=0, settled=true] (258) "\x00Sp\xd0\x00\x00\x00\x05\x00\x00\x00\x01B\x00Sr\xd1\x00\x00\x00_\x00\x00\x00\x08\xa3\x0ex-opt-qd.trace\xd0\x00\x00\x00\x13\x00\x00\x00\x03\xa1\x030/C\xa1\x030/B\xa1\x030/A\xa3\x10x-opt-qd.ingress\xa1\x030/C\xa3\x09x-opt-qd.\xa1\x01X\xa3\x09x-opt-qd.\xa1\x01X\x00Ss\xd0\x00\x00\x00%\x00\x00\x00\x06@@\xa1\x1aamqp:/_topo/0/all/qdrouter@@@\x00St\xd1\x00\x00\x00\x10\x00\x00\x00\x02\xa1\x06opcode\xa1\x02RA\x00Sw\xd1\x00\x00\x00A\x00\x00\x00\x0c\xa1\x06ls_seqT\x01\xa1\x02pvT\x01\xa1\x04area\xa1\x010\xa1\x08instanceq[dm\xd7\xa1\x0amobile_seqT\x00\xa1\x02id\xa1\x01C" (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-08-03 10:59:48.006844 -0400 SERVER (trace) [6]:0 -> @transfer(20) [handle=0, delivery-id=447, delivery-tag=b""\x02\x00\x00\x00\x00\x00\x00", message-format=0, settled=true] (208) "\x00Sp\xd0\x00\x00\x00\x05\x00\x00\x00\x01B\x00Sr\xd1\x00\x00\x00U\x00\x00\x00\x08\xa3\x0ex-opt-qd.trace\xd0\x00\x00\x00\x09\x00\x00\x00\x01\xa1\x030/A\xa3\x10x-opt-qd.ingress\xa1\x030/A\xa3\x09x-opt-qd.\xa1\x01X\xa3\x09x-opt-qd.\xa1\x01X\x00Ss\xd0\x00\x00\x00#\x00\x00\x00\x06@@\xa1\x18amqp:/_topo/0/D/qdrouter@@@\x00St\xd1\x00\x00\x00\x11\x00\x00\x00\x02\xa1\x06opcode\xa1\x03LSR\x00Sw\xd1\x00\x00\x00\x1a\x00\x00\x00\x06\xa1\x02pvT\x01\xa1\x02id\xa1\x01A\xa1\x04area\xa1\x010" (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-08-03 10:59:49.480073 -0400 SERVER (trace) [7]:0 -> @transfer(20) [handle=0, delivery-id=3, delivery-tag=b"\x8b\x01\x00\x00\x00\x00\x00\x00", message-format=0, settled=true] (233) "\x00Sp\xd0\x00\x00\x00\x05\x00\x00\x00\x01B\x00Sr\xd1\x00\x00\x00U\x00\x00\x00\x08\xa3\x0ex-opt-qd.trace\xd0\x00\x00\x00\x09\x00\x00\x00\x01\xa1\x030/C\xa3\x10x-opt-qd.ingress\xa1\x030/C\xa3\x09x-opt-qd.\xa1\x01X\xa3\x09x-opt-qd.\xa1\x01X\x00Ss\xd0\x00\x00\x00/\x00\x00\x00\x06@@\xa1$amqp:/_topo/0/C/temp.y0iWM_zBNSbDane@@@\x00St\xd1\x00\x00\x004\x00\x00\x00\x04\xa1\x11statusDescription\xa1\x0aNo Content\xa1\x0astatusCodeq\x00\x00\x00\xcc\x00Sw\xd1\x00\x00\x00\x04\x00\x00\x00\x00" (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-08-03 10:59:43.485362 -0400 SERVER (trace) [8]:0 -> @transfer(20) [handle=0, delivery-id=3, delivery-tag=b"o\x00\x00\x00\x00\x00\x00\x00", message-format=0, settled=true] (1589) "\x00Sp\xd0\x00\x00\x00\x05\x00\x00\x00\x01B\x00Ss\xd0\x00\x00\x00/\x00\x00\x00\x06@@\xa1$amqp:/_topo/0/D/temp._HHAoiYZ39HlEEH@@@\x00St\xd1\x00\x00\x00,\x00\x00\x00\x04\xa1\x11statusDescription\xa1\x02OK\xa1\x0astatusCodeq\x00\x00\x00\xc8\x00Sw\xd1\x00\x00\x05\xb5\x00\x00\x00\x04\xa1\x0eattributeNames\xd0\x00\x00\x00\x95\x00\x00\x00\x0b\xa1\x08linkType\xa1\x07linkDir\xa1\x08linkName\xa1\x0aowningAddr\xa1\x08capacity\xa1\x10undeliveredCount\xa1\x0eunsettledCount\xa1\x0dacceptedCount\xa1\x0drejectedCount\xa1\x0dreleasedCount\xa1\x0dmodifiedCount\xa1\x07results\xd0\x00\x00\x04\xf9\x00\x00\x00\x10\xd0\x00\x00\x00<\x00\x00\x00\x0b\xa1\x0erouter-control\xa1\x02in\xa1\x16qdlink.EZD43Jm5VvSht0w@p\x00\x00\x03\xe8DDDDDD\xd0\x00\x00\x00F\x00\x00\x00\x0b\xa1\x0erouter-control\xa1\x03out\xa1\x16qdlink.STppD563DOcP2Z
R\xa1\x08Lqdhellop\x00\x00\x03\xe8DDDDDD\xd0\x00\x00\x00:\x00\x00\x00\x0b\xa1\x0cinter-router\xa1\x02in\xa1\x16qdlink.inIy3q1zJObSUhB@p\x00\x00\x03\xe8DDDDDD\xd0\x00\x00\x00<\x00\x00\x00\x0b\xa1\x0cinter-route"... (truncated) (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-08-06 13:41:42.793480 -0400 SERVER (trace) [7]:0 -> (EMPTY FRAME)
-2018-08-16 13:53:22.276203 -0400 SERVER (trace) [22]:0 <- @attach(18) [name="6216f14e-16db-4000-89eb-9716c02ef9e3-1acf0c3a-bdaa-4ce2-8786-e39c65bdb7bf", handle=0, role=true, snd-settle-mode=2, rcv-settle-mode=0, source=@source(40) [durable=0, timeout=0, dynamic=true, dynamic-node-properties={:"x-opt-qd.address"="pulp.task.abc"}], target=@target(41) [durable=0, timeout=0, dynamic=false], initial-delivery-count=0, max-message-size=0] (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-08-16 13:56:59.038310 -0400 SERVER (trace) [8]:0 -> @transfer(20) [handle=0, delivery-id=1, delivery-tag=b"\x14\x00\x00\x00\x00\x00\x00\x00", message-format=0, settled=true, aborted=true] (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-08-24 14:29:26.821739 -0400 SERVER (trace) [2]:0 <- @attach(18) [name="qdlink.YoQaLsapwDzqhOL", handle=1, role=true, snd-settle-mode=2, rcv-settle-mode=0, source=@source(40) [durable=0, expiry-policy=:"session-end", timeout=0, dynamic=false, capabilities=:"qd.router"], target=@target(41) [durable=0, expiry-policy=:"session-end", timeout=0, dynamic=false, capabilities=:"qd.router"], initial-delivery-count=0, max-message-size=0] (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-08-24 14:29:26.821750 -0400 SERVER (trace) [1]:0 <- @attach(18) [name="qdlink.qvRSF0ysELu13cM", handle=2, role=false, snd-settle-mode=2, rcv-settle-mode=0, source=@source(40) [durable=0, expiry-policy=:"session-end", timeout=0, dynamic=false, capabilities=:"qd.router-data"], target=@target(41) [durable=0, expiry-policy=:"session-end", timeout=0, dynamic=false, capabilities=:"qd.router-data"], initial-delivery-count=0, max-message-size=0] (/home/chug/git/qpid-dispatch/src/server.c:106)
-2018-10-11 14:55:31.302512 -0400 SERVER (trace) [4]:0 -> @open(16) [container-id="A", max-frame-size=16384, channel-max=32767, idle-time-out=60000, offered-capabilities=:"ANONYMOUS-RELAY", properties={:product="qpid-dispatch-router", :version="1.4.0-SNAPSHOT", :"qd.conn-id"=4}] (/home/chug/git/qpid-dispatch/src/server.c:106)
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/b7ab3390/bin/log_scraper/text.py
----------------------------------------------------------------------
diff --git a/bin/log_scraper/text.py b/bin/log_scraper/text.py
deleted file mode 100755
index ba6a1f8..0000000
--- a/bin/log_scraper/text.py
+++ /dev/null
@@ -1,137 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-"""Common text strings"""
-
-
-def direction_in():
- """Log line text indicating received by router"""
- return "<-"
-
-
-def direction_out():
- """Log line text indicating transmitted by router"""
- return "->"
-
-
-def lozenge():
- """
- :return: HTML document lozenge character
- """
- return "◊"
-
-
-def nbsp():
- """
- :return: HTML Non-breaking space
- """
- return " "
-
-
-"""Large text strings used by main that change infrequently"""
-
-
-# html head, start body
-def web_page_head():
- return """<!DOCTYPE html>
-<html>
-<head>
-<title>Adverbl Analysis - qpid-dispatch router logs</title>
-
-<style>
- * {
- font-family: sans-serif;
-}
-table {
- border-collapse: collapse;
-}
-table, td, th {
- border: 1px solid black;
- padding: 3px;
-}
-</style>
-
-<script src="http://ajax.googleapis.com/ajax/libs/dojo/1.4/dojo/dojo.xd.js" type="text/javascript"></script>
-<!-- <script src="http://ajax.googleapis.com/ajax/libs/dojo/1.4/dojo/dojo.xd.js" type="text/javascript"></script> -->
-<script type="text/javascript">
-function node_is_visible(node)
-{
- if(dojo.isString(node))
- node = dojo.byId(node);
- if(!node)
- return false;
- return node.style.display == "block";
-}
-function set_node(node, str)
-{
- if(dojo.isString(node))
- node = dojo.byId(node);
- if(!node) return;
- node.style.display = str;
-}
-function toggle_node(node)
-{
- if(dojo.isString(node))
- node = dojo.byId(node);
- if(!node) return;
- set_node(node, (node_is_visible(node)) ? 'none' : 'block');
-}
-function hide_node(node)
-{
- set_node(node, 'none');
-}
-function show_node(node)
-{
- set_node(node, 'block');
-}
-
-function go_back()
-{
- window.history.back();
-}
-"""
-
-
-def web_page_toc():
- return """
-<h3>Contents</h3>
-<table>
-<tr> <th>Section</th> <th>Description</th> </tr>
-<tr><td><a href=\"#c_logfiles\" >Log files</a></td> <td>Router and log file info</td></tr>
-<tr><td><a href=\"#c_rtrinstances\" >Router Instances</a></td> <td>Router reboot chronology</td></tr>
-<tr><td><a href=\"#c_connections\" >Connections</a></td> <td>Connection overview; per connection log data view control</td></tr>
-<tr><td><a href=\"#c_conndetails\" >Connection Details</a></td> <td>Connection details; frames sorted by link</td></tr>
-<tr><td><a href=\"#c_noteworthy\" >Noteworthy log lines</a></td> <td>AMQP errors and interesting flags</td></tr>
-<tr><td><a href=\"#c_logdata\" >Log data</a></td> <td>Main AMQP traffic table</td></tr>
-<tr><td><a href=\"#c_messageprogress\">Message progress</a></td> <td>Tracking messages through the system</td></tr>
-<tr><td><a href=\"#c_linkprogress\" >Link name propagation</a></td> <td>Tracking link names</td></tr>
-<tr><td><a href=\"#c_rtrdump\" >Router name index</a></td> <td>Short vs. long router container names</td></tr>
-<tr><td><a href=\"#c_peerdump\" >Peer name index</a></td> <td>Short vs. long peer names</td></tr>
-<tr><td><a href=\"#c_linkdump\" >Link name index</a></td> <td>Short vs. long link names</td></tr>
-<tr><td><a href=\"#c_msgdump\" >Transfer name index</a></td> <td>Short names representing transfer data</td></tr>
-<tr><td><a href=\"#c_ls\" >Router link state</a></td> <td>Link state analysis</td></tr>
-</table>
-<hr>
-"""
-
-
-if __name__ == "__main__":
- pass
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/b7ab3390/tools/scraper/README.md
----------------------------------------------------------------------
diff --git a/tools/scraper/README.md b/tools/scraper/README.md
new file mode 100644
index 0000000..64ec977
--- /dev/null
+++ b/tools/scraper/README.md
@@ -0,0 +1,179 @@
+# Scraper - Render qpid-dispatch log files
+
+Scraper is a spinoff of https://github.com/ChugR/Adverb that uses qpid-dispatch log
+files as the data source instead of pcap trace files. Scraper is a Python processing
+engine that does not require Wireshark or any network trace capture utilities.
+
+## Apache License, Version 2.0
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License. You may obtain a copy of the
+License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed
+under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+
+## Concepts
+
+Scraper is a data scraping program. It reads qpid-dispatch router log files,
+categorizes and sorts the data, and produces an HTML summary.
+
+From each log file Scraper extracts:
+ * Router version
+ * Router container name
+ * Router restart times. A single log file may contain data from several router
+ reboot instances.
+ * Router link state calculation reports
+ * Interrouter and client connections
+ * AMQP facts
+ * Connection peers
+ * Link pair establishment
+ * Transfer traffic
+ * Message disposition
+ * Flow and credit propagation
+
+ Scraper sorts these facts with microsecond precision using the log timestamps.
+
+ Then Scraper merges the data from any number (as long as that number is less than 27!)
+ of independent log files into a single view.
+
+ Next Scraper performs some higher-level analysis.
+
+ * Routers are identified by letter rather than by the container name: 'A', 'B', and
+ so on. Log data in a file is grouped into instances and is identified by a number
+ for that router instance: 'A0', 'A1', and so on.
+ * Per router each AMQP data log entry is sorted into per-connection data lists.
+ * Connection data lists are searched to discover router-to-router and router-to-client
+ connection pairs.
+ * Per connection data are subdivided into per-session and per-link lists, sorting
+ the AMQP data into per-link-only views.
+ * Bulk AMQP data may be shown or hidden on arbitrary per-connection selections.
+ * Noteworthy AMQP frames are identified. By hand these are hard to find.
+ * AMQP errors
+ * Presettled transfers
+ * Transfers with 'more' bit set
+ * Resumed transfers
+ * Aborted transfers
+ * Flow with 'drain' set
+ * Transfer messages are sorted by signature. Then a table is made showing where
+ each message leaves or arrives over a connection.
+ * Settlement state for each unsettled transfer is identified, displayed, and
+ shown with delta and elapsed time values. See example in the Advanced section.
+ * Link name propagation for each named link is shown in a table.
+ * Router, peer, and link names can get really long. Nicknames for each are used
+ with popups showing the full name.
+ * Transfer data is identified with nicknames but without the popups. The popups
+ were so big that Firefox refused to show them; so forget it and they weren't useful anyway.
+ * Router link state cost calculations are merged with router restart records to
+ create a comprehensive link state cost view. Routers may publish cost reports that
+ do not include all other routers. In this case the other routers are identified
+ visually to indicate that they are unreachable.
+
+### The basics
+
+* Enable router logging
+
+The routers need to generate proper logging for Scraper.
+The information classes are exposed by enabling log levels.
+
+| Log level | Information |
+|----------------|---------------------------|
+| ROUTER info | Router version |
+| SERVER info | Router restart discovery |
+| SERVER trace | AMQP control and data |
+| ROUTER_LS info | Router link state reports |
+
+
+* Run your tests to populate log files used as Scraper input.
+
+* Run Scraper to generate web content
+
+ bin/scraper/main.py somefile.log > somefile.html
+
+ bin/scraper/mina.py *.log > somefile.html
+
+* Profit
+
+ firefox somefile.html
+
+### Advanced
+
+* Merging multiple qpid-dispatch log files
+
+Scraper accepts multiple log files names in the command line and
+merges the log data according to the router log timestamps.
+
+ bin/scraper/main.py A.log B.log C.log > abc.html
+
+Note that the qpid-dispatch host system clocks for merged log files
+must be synchronized to within a few microseconds in order for the
+result to be useful. This is easiest to achieve when the routers are
+run on the same CPU core on a single system. Running Fedora 27 and 28
+on two hosts in a router network where the routers run _ntp_ to the same
+time provider produces perfectly acceptable results.
+
+Scraper does a decent job merging log files created within a
+qpid-dispatch self test.
+
+* Wow, that's a lot of data
+
+Indeed it is and good luck figuring it out. Sometimes, though, it's too much.
+The AMQP transfer data analysis is the worst offender in terms of CPU time,
+run-time memory usage, and monstrous html output files.
+Scraper provides one command line switch to
+turn off the data analysis:
+
+ bin/scraper/main.py --no-data FILE [FILE ...]
+
+In no-data mode AMQP transfer, disposition, and flow frames in the log files are
+discarded. The resulting web page still includes lots of useful information with
+connection info, link name propagation, and link state analysis.
+
+* How to read the transfer analysis tables. Here's an instance:
+
+
+|Src |Time |Rtr|ConnId|Dir|ConnId|Peer |T delta |T elapsed |Settlement |S elapsed
+|-------|---------------|---|------|---|------|------|---------|----------|-----------------------------|---------
+|A0_2035|09:50:52.027975|A0 |A0_11 |<- | |peer_7|0.000000 |0.000000 |(accepted settled 0.005142 S)|0.005142
+|A0_2071|09:50:52.028556|A0 |A0_6 |-> |D0_4 |D |0.000581 |0.000581 |(accepted settled 0.004253 S)|0.004834
+|D0_1587|09:50:52.028696|D0 |D0_4 |<- |A0_6 |A |0.000140 |0.000721 |(accepted settled 0.003988 S)|0.004709
+|D0_1612|09:50:52.029260|D0 |D0_1 |-> |C0_6 |C |0.000564 |0.001285 |(accepted settled 0.003044 S)|0.004329
+|C0_1610|09:50:52.029350|C0 |C0_6 |<- |D0_1 |D |0.000090 |0.001375 |(accepted settled 0.002846 S)|0.004221
+|C0_1625|09:50:52.029672|C0 |C0_1 |-> |B0_5 |B |0.000322 |0.001697 |(accepted settled 0.002189 S)|0.003886
+|B0_1438|09:50:52.029760|B0 |B0_5 |<- |C0_1 |C |0.000088 |0.001785 |(accepted settled 0.002002 S)|0.003787
+|B0_1451|09:50:52.030117|B0 |B0_7 |-> | |peer_7|0.000357 |0.002142 |(accepted settled 0.001318 S)|0.003460
+
+Each row in this table represents the facts about when a single transfer and its corresponding settlement was seen entering or exiting a router.
+
+| Field | Contents |
+|--------------|----------|
+|Src | Router instance and file line number where the transfer was seen|
+| Time | timestamp
+| Rtr | Router letter id and instance
+| ConnId | Router connection id
+| Dir | transfer direction. _<-_ indicates into the router, _->_ indicates out of the router
+| ConnId | peer's connection id. Blank if the peer is a normal client and not a router.
+| Peer | Peer's name. _peer7_ whold show the peer's container name in a popup.
+| T delta | Time since previous row
+| T elapsed | Time since the message first entered the system
+| Settlement | Settlement state and time delta since message time in column 2 for this row. The settlement disposition log line is hyperlinked from the word _accepted_.
+| S elapsed | Settlement elapsed time. This is the difference between the accepted disposition log record and the time when the message first entered the system.
+
+Row-by-row it is easiest to read the each line from left to right
+* A0 connecton 11 received the transfer from peer_7.
+* A0 connection 6 sent the message to D0 connection 4.
+* D0 connection 4 received the message from A0 connection 6.
+
+and so on. This message came from a sender on peer_7, went through routers A, D, C, and B, and finally was
+returned to a listener on peer_7. The receiver received the message 0.002142 S after the sender sent it. The
+sender received the accepted disposition 0.005142 S after the sender sent the message.
+
+The transmit times are in order from top to bottom and the settlement times are in order from bottom to top.
+
+This table will morph a little if one of the router is missing from the analysis. If log file D.log was not
+presented to Scraper then the table would not make as much sense as when all logs are included.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org
For additional commands, e-mail: commits-help@qpid.apache.org