You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by bb...@apache.org on 2021/06/22 16:30:20 UTC
[geode-native] branch develop updated: GEODE-9387: Ignore version
trace in gnmsg (#822)
This is an automated email from the ASF dual-hosted git repository.
bbender pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode-native.git
The following commit(s) were added to refs/heads/develop by this push:
new f6b0ef6 GEODE-9387: Ignore version trace in gnmsg (#822)
f6b0ef6 is described below
commit f6b0ef6634ebf9f8fa58e04478fe47307a238167
Author: Blake Bender <bb...@pivotal.io>
AuthorDate: Tue Jun 22 09:29:49 2021 -0700
GEODE-9387: Ignore version trace in gnmsg (#822)
- makes the tool robust against weird versions of NC
- no longer any need to combine rolled log files to parse messages
---
tools/gnmsg/chunked_message_decoder.py | 35 ++++++++++----
tools/gnmsg/client_message_decoder.py | 67 +++++++++------------------
tools/gnmsg/server_message_decoder.py | 85 +++++++++++++---------------------
3 files changed, 81 insertions(+), 106 deletions(-)
diff --git a/tools/gnmsg/chunked_message_decoder.py b/tools/gnmsg/chunked_message_decoder.py
index e2be4b3..ef3330c 100644
--- a/tools/gnmsg/chunked_message_decoder.py
+++ b/tools/gnmsg/chunked_message_decoder.py
@@ -21,24 +21,33 @@ from read_values import read_int_value, read_byte_value, call_reader_function
CHUNKED_MESSAGE_HEADER_LENGTH = 17
+
class ChunkedResponseDecoder:
def __init__(self):
self.reset()
def add_header(self, connection, header):
if len(self.chunked_message) > 0:
- raise Exception("Previous chunked message is not completed, can't process another header")
+ raise Exception(
+ "Previous chunked message is not completed, can't process another header"
+ )
if len(header) == 2 * CHUNKED_MESSAGE_HEADER_LENGTH:
offset = 0
message_type = ""
- (message_type, offset) = call_reader_function(header, offset, read_int_value)
+ (message_type, offset) = call_reader_function(
+ header, offset, read_int_value
+ )
self.chunked_message["Type"] = message_types[message_type]
# TODO: pass connection value in as a parameter
self.chunked_message["Connection"] = connection
self.chunked_message["Direction"] = "<---"
- (self.chunked_message["Parts"], offset) = call_reader_function(header, offset, read_int_value)
- (self.chunked_message["TransactionId"], offset) = call_reader_function(header, offset, read_int_value)
+ (self.chunked_message["Parts"], offset) = call_reader_function(
+ header, offset, read_int_value
+ )
+ (self.chunked_message["TransactionId"], offset) = call_reader_function(
+ header, offset, read_int_value
+ )
chunk_size = 0
flags = 0
(chunk_size, offset) = call_reader_function(header, offset, read_int_value)
@@ -46,7 +55,11 @@ class ChunkedResponseDecoder:
self.chunked_message["ChunkInfo"] = []
self.add_chunk_header(chunk_size, flags)
else:
- raise IndexError("Chunked message header should be " + str(CHUNKED_MESSAGE_HEADER_LENGTH) + " bytes")
+ raise IndexError(
+ "Chunked message header should be "
+ + str(CHUNKED_MESSAGE_HEADER_LENGTH)
+ + " bytes"
+ )
def add_chunk_header(self, chunk_size, flags):
self.chunk_index += 1
@@ -69,13 +82,17 @@ class ChunkedResponseDecoder:
def is_complete_message(self):
return self.chunk_flags & 0x1
- def get_decoded_message(self):
- return self.chunked_message
+ def get_decoded_message(self, time_stamp):
+ # Return a re-ordered dictionary, with Timestamp at the front. This
+ # makes output consistent with other messages
+ decoded_message = {"Timestamp": time_stamp}
+ decoded_message.update(self.chunked_message)
+ return decoded_message
def reset(self):
self.header = ""
self.message_body = ""
self.chunked_message = {}
self.complete = False
- self.chunk_flags = 0xff
- self.chunk_index = -1
\ No newline at end of file
+ self.chunk_flags = 0xFF
+ self.chunk_index = -1
diff --git a/tools/gnmsg/client_message_decoder.py b/tools/gnmsg/client_message_decoder.py
index 23dbb4d..45e88f7 100644
--- a/tools/gnmsg/client_message_decoder.py
+++ b/tools/gnmsg/client_message_decoder.py
@@ -33,30 +33,15 @@ class ClientMessageDecoder(DecoderBase):
self.send_trace_parts_retriever_ = None
self.send_trace_parser_ = None
self.connection_states_ = {}
- self.nc_version_ = None
- self.send_trace_parts_retriever_ = None
- self.get_send_trace_parts_functions = {
- "0.0.42": self.get_send_trace_parts_base,
- "10.0.3": self.get_send_trace_parts_base,
- "10.1.1": self.get_send_trace_parts_base,
- "10.1.2": self.get_send_trace_parts_base,
- "10.1.3": self.get_send_trace_parts_base,
- "10.1.4": self.get_send_trace_parts_base,
- "10.2.0": self.get_send_trace_parts_base,
- "10.2.1": self.get_send_trace_parts_base,
- "9.1.1": self.get_send_trace_parts_v911,
- }
- self.send_trace_parsers = {
- "0.0.42": self.parse_request_fields_base,
- "10.0.3": self.parse_request_fields_base,
- "10.1.1": self.parse_request_fields_base,
- "10.1.2": self.parse_request_fields_base,
- "10.1.3": self.parse_request_fields_base,
- "10.1.4": self.parse_request_fields_base,
- "10.2.0": self.parse_request_fields_base,
- "10.2.1": self.parse_request_fields_base,
- "9.1.1": self.parse_request_fields_v911,
- }
+ self.get_send_trace_parts_functions = [
+ self.get_send_trace_parts_base,
+ self.get_send_trace_parts_v911,
+ ]
+ self.send_trace_parsers = [
+ self.parse_request_fields_base,
+ self.parse_request_fields_v911,
+ ]
+
#
# Native client code believes this is the list of messages that require a security footer.
# We will use this list to verify and report if a message is sent that needs one but doesn't
@@ -92,28 +77,14 @@ class ClientMessageDecoder(DecoderBase):
"USER_CREDENTIAL_MESSAGE",
]
- def search_for_version(self, line):
- if self.nc_version_ == None:
- expression = re.compile(r"Product version:.*Native (\d+)\.(\d+)\.(\d+)-")
- match = expression.search(line)
- if match:
- major = match.group(1)
- minor = match.group(2)
- patch = match.group(3)
- self.nc_version_ = major + "." + minor + "." + patch
- self.send_trace_parts_retriever_ = self.get_send_trace_parts_functions[
- self.nc_version_
- ]
- self.send_trace_parser_ = self.send_trace_parsers[self.nc_version_]
-
def get_send_trace_parts_v911(self, line, parts):
result = False
expression = re.compile(
- r"(\d\d:\d\d:\d\d\.\d+).*TcrConnection::send:\s*\[([\d|a-f|A-F|x|X]+).*sending request to endpoint.*bytes:\s*([\d| ]+)"
+ r"(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d\.\d+).*TcrConnection::send:\s*\[([\d|a-f|A-F|x|X]+).*sending request to endpoint.*bytes:\s*([\d| ]+)"
)
match = expression.search(line)
if match:
- parts.append(dateutil.parser.parse(match.group(1)))
+ parts.append(parser.parse(match.group(1)))
# TODO: Revisit parsing TID here if we ever see a v9 client log again
parts.append("0")
parts.append(match.group(2))
@@ -125,7 +96,7 @@ class ClientMessageDecoder(DecoderBase):
def get_send_trace_parts_base(self, line, parts):
result = False
expression = re.compile(
- r"(\d\d:\d\d:\d\d\.\d+).+:\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::send:\s*\[([\d|a-f|A-F|x|X]+).*sending request to endpoint.*bytes:\s*([\d|a-f|A-F]+)"
+ r"(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d\.\d+).+:\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::send:\s*\[([\d|a-f|A-F|x|X]+).*sending request to endpoint.*bytes:\s*([\d|a-f|A-F]+)"
)
match = expression.search(line)
if match:
@@ -140,11 +111,21 @@ class ClientMessageDecoder(DecoderBase):
def get_send_trace_parts(self, line, parts):
if self.send_trace_parts_retriever_ is not None:
return self.send_trace_parts_retriever_(line, parts)
+ else:
+ for retriever in self.get_send_trace_parts_functions:
+ if retriever(line, parts):
+ self.send_trace_parts_retriever_ = retriever
+ self.send_trace_parser_ = self.send_trace_parsers[
+ self.get_send_trace_parts_functions.index(retriever)
+ ]
+ return True
+ else:
+ return False
def get_add_security_trace_parts(self, line, parts):
result = False
expression = re.compile(
- r"(\d\d:\d\d:\d\d\.\d+).*([\d|a-f|A-F|x|X]+)\]\s*TcrMessage::addSecurityPart\s*\[(0x[\d|a-f|A-F]*).*length\s*=\s*(\d+)\s*,\s*encrypted\s+ID\s*=\s*([\d|a-f|A-F]+)"
+ r"(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d\.\d+).*([\d|a-f|A-F|x|X]+)\]\s*TcrMessage::addSecurityPart\s*\[(0x[\d|a-f|A-F]*).*length\s*=\s*(\d+)\s*,\s*encrypted\s+ID\s*=\s*([\d|a-f|A-F]+)"
)
match = expression.search(line)
if match:
@@ -216,8 +197,6 @@ class ClientMessageDecoder(DecoderBase):
is_add_security_trace = False
send_trace = {}
- self.search_for_version(line)
-
parts = []
if self.get_send_trace_parts(line, parts):
(
diff --git a/tools/gnmsg/server_message_decoder.py b/tools/gnmsg/server_message_decoder.py
index dfb9df9..22e9265 100644
--- a/tools/gnmsg/server_message_decoder.py
+++ b/tools/gnmsg/server_message_decoder.py
@@ -1,4 +1,5 @@
-#!/usr/local/bin/python3
+# get_send_trace_parts_functionsrse
+# !/usr/local/bin/python3
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -38,47 +39,17 @@ class ServerMessageDecoder(DecoderBase):
self.connection_states_ = {}
self.headers_ = {}
self.nc_version_ = None
- self.get_receive_trace_parts_functions_ = {
- "0.0.42": self.get_receive_trace_header_base,
- "10.0.3": self.get_receive_trace_header_base,
- "10.1.1": self.get_receive_trace_header_base,
- "10.1.2": self.get_receive_trace_header_base,
- "10.1.3": self.get_receive_trace_header_base,
- "10.1.4": self.get_receive_trace_header_base,
- "10.2.0": self.get_receive_trace_header_base,
- "10.2.1": self.get_receive_trace_header_base,
- "9.1.1": self.get_receive_trace_header_v911,
- }
- self.receive_trace_parsers_ = {
- "0.0.42": self.parse_response_fields_base,
- "10.0.3": self.parse_response_fields_base,
- "10.1.1": self.parse_response_fields_base,
- "10.1.2": self.parse_response_fields_base,
- "10.1.3": self.parse_response_fields_base,
- "10.1.4": self.parse_response_fields_base,
- "10.2.0": self.parse_response_fields_base,
- "10.2.1": self.parse_response_fields_base,
- "9.1.1": self.parse_response_fields_v911,
- }
+ self.get_receive_trace_parts_functions_ = [
+ self.get_receive_trace_header_base,
+ self.get_receive_trace_header_v911,
+ ]
+ self.receive_trace_parsers_ = [
+ self.parse_response_fields_base,
+ self.parse_response_fields_v911,
+ ]
self.chunk_decoder = ChunkedResponseDecoder()
self.threads_connections_ = {}
- def search_for_version(self, line):
- if self.nc_version_ == None:
- expression = re.compile(r"Product version:.*Native (\d+)\.(\d+)\.(\d+)-")
- match = expression.search(line)
- if match:
- major = match.group(1)
- minor = match.group(2)
- patch = match.group(3)
- self.nc_version_ = major + "." + minor + "." + patch
- self.receive_trace_parts_retriever_ = (
- self.get_receive_trace_parts_functions_[self.nc_version_]
- )
- self.receive_trace_parser_ = self.receive_trace_parsers_[
- self.nc_version_
- ]
-
def associate_connection_to_tid(self, line):
result = False
expression = re.compile(
@@ -96,11 +67,11 @@ class ServerMessageDecoder(DecoderBase):
def get_receive_trace_header_with_pointer(self, line, parts):
result = False
expression = re.compile(
- r"(\d\d:\d\d:\d\d\.\d+).+:\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::readMessage\(([\d|a-f|A-F|x|X]+)\):.*received header from endpoint.*bytes:\s*([\d|a-f|A-F]+)"
+ r"(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d\.\d+).+:\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::readMessage\(([\d|a-f|A-F|x|X]+)\):.*received header from endpoint.*bytes:\s*([\d|a-f|A-F]+)"
)
match = expression.search(line)
if match:
- date_time = match.group(1)
+ date_time = parser.parse(match.group(1))
tid = match.group(2)
connection = match.group(3)
bytes = match.group(4)
@@ -116,12 +87,12 @@ class ServerMessageDecoder(DecoderBase):
def get_receive_trace_header_without_pointer(self, line, parts):
result = False
expression = re.compile(
- r"(\d\d:\d\d:\d\d\.\d+).*:\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::readMessage:\s*received header from endpoint.*bytes:\s*([\d|a-f|A-F]+)"
+ r"(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d\.\d+).*:\d+\s+([\d|a-f|A-F|x|X]+)\]\s*TcrConnection::readMessage:\s*received header from endpoint.*bytes:\s*([\d|a-f|A-F]+)"
)
match = expression.search(line)
if match:
tid = match.group(2)
- parts.append(match.group(1))
+ parts.append(parser.parse(match.group(1)))
parts.append(tid)
if tid in self.threads_connections_.keys():
parts.append(self.threads_connections_[tid])
@@ -142,7 +113,7 @@ class ServerMessageDecoder(DecoderBase):
def get_receive_trace_header_v911(self, line, parts):
result = False
expression = re.compile(
- r"(\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readMessage: received header from endpoint.*bytes:\s*([\d| ]+)"
+ r"(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readMessage: received header from endpoint.*bytes:\s*([\d| ]+)"
)
match = expression.search(line)
if match:
@@ -179,11 +150,21 @@ class ServerMessageDecoder(DecoderBase):
def get_receive_trace_parts(self, line, parts):
if self.receive_trace_parts_retriever_ is not None:
return self.receive_trace_parts_retriever_(line, parts)
+ else:
+ for retriever in self.get_receive_trace_parts_functions_:
+ if retriever(line, parts):
+ self.receive_trace_parts_retriever_ = retriever
+ self.receive_trace_parser_ = self.receive_trace_parsers_[
+ self.get_receive_trace_parts_functions_.index(retriever)
+ ]
+ return True
+ else:
+ return False
def get_add_security_trace_parts(self, line, parts):
result = False
expression = re.compile(
- r"(\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrMessage::addSecurityPart\s*\[(0x[\d|a-f|A-F]*).*length\s*=\s*(\d+)\s*,\s*encrypted\s+ID\s*=\s*([\d|a-f|A-F]+)"
+ r"(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrMessage::addSecurityPart\s*\[(0x[\d|a-f|A-F]*).*length\s*=\s*(\d+)\s*,\s*encrypted\s+ID\s*=\s*([\d|a-f|A-F]+)"
)
match = expression.search(line)
if match:
@@ -201,7 +182,7 @@ class ServerMessageDecoder(DecoderBase):
result = False
expression = re.compile(
- r"(\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readResponseHeader\(([0-9|a-f|A-F|x]+)\):\s*received header from endpoint\s*([\w|:|\d|\.|-]+);\s*bytes:\s*([\d|a-f|A-F]+)"
+ r"(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readResponseHeader\(([0-9|a-f|A-F|x]+)\):\s*received header from endpoint\s*([\w|:|\d|\.|-]+);\s*bytes:\s*([\d|a-f|A-F]+)"
)
match = expression.search(line)
if match:
@@ -217,7 +198,7 @@ class ServerMessageDecoder(DecoderBase):
if not result:
expression = re.compile(
- r"(\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readResponseHeader:\s*received header from endpoint\s*([\w|:|\d|\.|-]+);\s*bytes:\s*([\d|a-f|A-F]+)"
+ r"(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readResponseHeader:\s*received header from endpoint\s*([\w|:|\d|\.|-]+);\s*bytes:\s*([\d|a-f|A-F]+)"
)
match = expression.search(line)
if match:
@@ -234,7 +215,7 @@ class ServerMessageDecoder(DecoderBase):
# Check if this is a header for a chunked message
result = False
expression = re.compile(
- r"(\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readChunkHeader\(([0-9|a-f|A-F|x]+)\):\s*.*, chunkLen=(\d+), lastChunkAndSecurityFlags=([0-9|a-f|A-F|x]+)"
+ r"(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readChunkHeader\(([0-9|a-f|A-F|x]+)\):\s*.*, chunkLen=(\d+), lastChunkAndSecurityFlags=([0-9|a-f|A-F|x]+)"
)
match = expression.search(line)
if match:
@@ -250,7 +231,7 @@ class ServerMessageDecoder(DecoderBase):
if not result:
expression = re.compile(
- r"(\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readChunkHeader:\s*.*, chunkLen=(\d+), lastChunkAndSecurityFlags=([0-9|a-f|A-F|x]+)"
+ r"(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readChunkHeader:\s*.*, chunkLen=(\d+), lastChunkAndSecurityFlags=([0-9|a-f|A-F|x]+)"
)
match = expression.search(line)
if match:
@@ -285,7 +266,7 @@ class ServerMessageDecoder(DecoderBase):
if not result:
expression = re.compile(
- r"(\d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readChunkBody: \s*received chunk body from endpoint\s*([\w|:|\d|\.|-]+);\s*bytes:\s*([\d|a-f|A-F]+)"
+ r"(\d\d\d\d\/\d\d\/\d\d \d\d:\d\d:\d\d\.\d+).*:\d+\s+(\d+)\]\s*TcrConnection::readChunkBody: \s*received chunk body from endpoint\s*([\w|:|\d|\.|-]+);\s*bytes:\s*([\d|a-f|A-F]+)"
)
match = expression.search(line)
if match:
@@ -354,8 +335,6 @@ class ServerMessageDecoder(DecoderBase):
chunk_bytes = None
tid = None
- self.search_for_version(line)
-
parts = []
if self.associate_connection_to_tid(line):
pass
@@ -393,7 +372,7 @@ class ServerMessageDecoder(DecoderBase):
self.chunk_decoder.add_chunk(parts[3])
if self.chunk_decoder.is_complete_message():
self.output_queue_.put(
- {"message": self.chunk_decoder.get_decoded_message()}
+ {"message": self.chunk_decoder.get_decoded_message(parts[0])}
)
self.chunk_decoder.reset()
else: