[incubator-ponymail-unit-tests] 01/02: Initial readme and main runall program

[hu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-unit-tests.git

commit c8f0cf0a49b33354bf57286e6d60db6a155c50a6
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Tue Aug 11 13:07:30 2020 +0200

    Initial readme and main runall program
    
    Actual test cases and programs to follow.
---
 README.md | 15 +++++++++++++++
 runall.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..bb678b8
--- /dev/null
+++ b/README.md
@@ -0,0 +1,15 @@
+# Apache Pony Mail Unit Tests Repository
+
+The aim of this repository is to create a comprehensive suite of tests for ensuring 
+that changes to the Pony Mail codebase does not impact stability and reproducibility.
+
+The repository is split into three main directories:
+
+- `tests/`: The python test scripts
+- `yaml/`: The test specifications
+- `corpus/`: The test corpus (data input to be used during tests)
+
+The root directory has a `runall.py`, which will run all tests it can find in the 
+yaml directory, and summarize the results at the end. You may also run individual 
+tests from the tests directory (more on that as we build out the test dir).
+
diff --git a/runall.py b/runall.py
new file mode 100644
index 0000000..7680f74
--- /dev/null
+++ b/runall.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import subprocess
+import argparse
+import yaml
+import time
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Command line options.')
+    parser.add_argument('--rootdir', dest='rootdir', type=str, required=True,
+                        help="Root directory of Apache Pony Mail")
+    parser.add_argument('--load', dest='load', type=str,
+                        help="Load only a specific yaml spec instead of all test specs")
+    args = parser.parse_args()
+
+    if args.load:
+        spec_files = [args.load]
+    else:
+        spec_files = [os.path.join('yaml', x) for x in os.listdir('yaml') if x.endswith('.yaml')]
+
+    tests_success = 0
+    tests_failure = 0
+    tests_total = 0
+    now = time.time()
+
+    for spec_file in spec_files:
+        with open(spec_file, 'r') as f:
+            yml = yaml.safe_load(f)
+            for test_type in yml:
+                tests_total += 1
+                print("Running '%s' tests from %s..." % (test_type, spec_file))
+                try:
+                    subprocess.check_call(('/usr/bin/python3', 'tests/test-%s.py' % test_type, '--rootdir', args.rootdir, '--load', spec_file))
+                    tests_success += 1
+                except subprocess.CalledProcessError as e:
+                    print("%s test from %s failed with code %d" % (test_type, spec_file, e.returncode))
+                    tests_failure += 1
+
+    print("-------------------------------------")
+    print("Done with %u tests in %.2f seconds" % (tests_total, time.time() - now))
+    print("%u Were GOOD, %u were BAD" % (tests_success, tests_failure))
+    print("-------------------------------------")
+    if tests_failure:
+        sys.exit(-1)