You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ta...@apache.org on 2021/11/08 13:24:48 UTC
[iotdb] 01/01: add tools
This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch e2e_parser
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit db8b616384435952e533915517bda232412c195d
Author: LebronAl <TX...@gmail.com>
AuthorDate: Mon Nov 8 21:23:00 2021 +0800
add tools
---
testcontainer/src/tool/README.md | 33 +++++++++++++++++++
testcontainer/src/tool/parser.py | 68 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 101 insertions(+)
diff --git a/testcontainer/src/tool/README.md b/testcontainer/src/tool/README.md
new file mode 100644
index 0000000..3b943af
--- /dev/null
+++ b/testcontainer/src/tool/README.md
@@ -0,0 +1,33 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+## Motivation
+The current E2E framework writes all process logs into a single test log. This approach makes it difficult to find the root cause when there are failed tests with unstable recurrence. So we need a tool that can find the failed tests in this log and separate the logs from the different nodes.
+
+## Usage
+1. Download log archive from CI.
+
+2. Parse log.
+```
+python3 parser.py [filename]
+```
+
+3. View the separated logs in the current directory.
\ No newline at end of file
diff --git a/testcontainer/src/tool/parser.py b/testcontainer/src/tool/parser.py
new file mode 100644
index 0000000..2f68e42
--- /dev/null
+++ b/testcontainer/src/tool/parser.py
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+import sys
+import re
+
+pattern = re.compile(r'docker\-java\-stream\-+(\d+)')
+
+
+def getAllLogs(filename):
+ with open(filename, "r") as f:
+ data = f.read()
+ return data.split("up -d")[1:]
+
+
+def writeAllLogs(filename, content):
+ with open(filename, "w") as f:
+ for row in content:
+ f.write(row)
+ f.write("\n")
+
+
+def getNodes(log):
+ ids = pattern.findall(log)
+ nodes = {}
+ for id in ids:
+ if not nodes.__contains__(id):
+ nodes[id] = []
+ return nodes
+
+
+if __name__ == "__main__":
+ logs = getAllLogs(sys.argv[1])
+ count = 0
+ for i in range(len(logs)):
+ if logs[i].__contains__("FAILURE!"):
+ nodes = getNodes(logs[i])
+ rows = logs[i].split("\n")
+ for row in rows:
+ for id, content in nodes.items():
+ if row.__contains__(id):
+ content.append(row)
+ if row.__contains__("[ERROR]"):
+ for content in nodes.values():
+ content.append(row)
+ count = count + 1
+ for key, content in nodes.items():
+ if not os.path.exists("./{}".format(i)):
+ os.mkdir("./{}".format(i))
+ writeAllLogs("./{}/{}_{}.txt".format(i, i, key), content)
+
+ print("find {} failed tests".format(count))