You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ponymail.apache.org by hu...@apache.org on 2021/06/02 12:23:16 UTC
[incubator-ponymail-unit-tests] 02/05: --dropin for quickly
reconfiguring unit tests on the fly
This is an automated email from the ASF dual-hosted git repository.
humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-unit-tests.git
commit fb8025f06013811eed9e2ea3e4f7946dde1eea16
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Wed Jun 2 14:02:14 2021 +0200
--dropin for quickly reconfiguring unit tests on the fly
---
runall.py | 7 ++++++-
tests/test-generators.py | 11 +++++++++++
2 files changed, 17 insertions(+), 1 deletion(-)
diff --git a/runall.py b/runall.py
index 556eec8..4b735cb 100755
--- a/runall.py
+++ b/runall.py
@@ -20,7 +20,10 @@ if __name__ == '__main__':
parser.add_argument('--gtype', dest='gtype', type=str, nargs='+',
help="Run only specified generators (medium, cluster, dkim, full, etc)")
parser.add_argument('--yamldir', dest='yamldir', type=str, action='store',
- help="Load yaml specs from alternate directory")
+ help="Load yaml specs from alternate directory")
+ parser.add_argument('--dropin', dest='dropin', type=str, action='store',
+ help="If set to a specific generator name, replaces its unit test results with the current "
+ "output in the yaml tests")
parser.add_argument('--nomboxo', dest = 'nomboxo', action='store_true',
help = 'Skip Mboxo processing')
parser.add_argument('--fof', dest='failonfail', action='store_true',
@@ -69,6 +72,8 @@ if __name__ == '__main__':
if args.gtype and test_type == 'generators':
cliargs.append('--generators')
cliargs.extend(args.gtype)
+ if args.dropin:
+ cliargs.extend(['--dropin', args.dropin])
if args.skipnodate and test_type == 'generators':
cliargs.append('--skipnodate')
rv = subprocess.check_output(cliargs, env=env)
diff --git a/tests/test-generators.py b/tests/test-generators.py
index aaa3daa..670d732 100755
--- a/tests/test-generators.py
+++ b/tests/test-generators.py
@@ -10,6 +10,7 @@ import yaml
import argparse
import collections
import interfacer
+import time
import email.utils
parse_html = False
@@ -150,9 +151,17 @@ def run_tests(args):
errors += 1
sys.stderr.write("""[FAIL] %s, index %2u: Expected '%s', got '%s'!\n""" %
(gen_type, key, expected, actual))
+ if args.dropin and gen_type == args.dropin:
+ if expected != actual:
+ test['generated'] = actual
+ else:
+ test['alternate'] = actual
else:
print("[PASS] %s index %u" % (gen_type, key))
mboxfiles = [] # reset for the next set of tests
+ if args.dropin and errors:
+ sys.stderr.write("Writing replacement yaml as --dropin was specified\n")
+ yaml.safe_dump(yml, open(args.load, "w"), sort_keys=False)
print("[DONE] %u tests run, %u failed." % (tests_run, errors))
if errors:
sys.exit(-1)
@@ -174,6 +183,8 @@ def main():
help="Root directory of Apache Pony Mail")
parser.add_argument('--nomboxo', dest = 'nomboxo', action='store_true',
help = 'Skip Mboxo processing')
+ parser.add_argument('--dropin', dest = 'dropin', type=str,
+ help = 'Perform drop-in replacement of unit test results for the specified generator type [devs only!]')
parser.add_argument('--skipnodate', dest = 'skipnodate', action='store_true',
help = 'Skip emails with no Date: header (useful for medium generator tests)')
args = parser.parse_args()
Re: [incubator-ponymail-unit-tests] 02/05: --dropin for quickly
reconfiguring unit tests on the fly
Posted by Daniel Gruno <hu...@apache.org>.
On 03/06/2021 01.33, sebb wrote:
> -1 to this change.
>
> It is not right to arbitrarily change test data this way.
>
> Changes to test data should only be made if there is a genuine reason to do so
> e.g. due to a mistake in the original data or a planned change in the API.
Yes, that is what it is intended for. It's a flag that you can use when
you fundamentally change a generator on purpose, like if we merge in the
new DKIM patch. It's basically there so we won't have to manually
recreate every single test file with the updated parameters, and that's
all I have intended for that flag.
I think you can say "I don't want this unit test result changed", but to
disallow the _option_ of changing a result programatically, I think is
not warranted. The dropin flag is akin to running each generate command
invidually but with the added safety net that it only changes the
specified generator results. Thus I can use it to change all DKIM
results if/when we change the DKIM generator, but leave everything else
as it was. I find it a useful flag.
>
> If a test starts failing, the most likely reason is a bug due to a
> change in the code, not a bug in the test data. >
> (There are of course exceptions, e.g. where a test is dependent on
> timing or other environmental factors)
>
> On Wed, 2 Jun 2021 at 13:23, <hu...@apache.org> wrote:
>>
>> This is an automated email from the ASF dual-hosted git repository.
>>
>> humbedooh pushed a commit to branch master
>> in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-unit-tests.git
>>
>> commit fb8025f06013811eed9e2ea3e4f7946dde1eea16
>> Author: Daniel Gruno <hu...@apache.org>
>> AuthorDate: Wed Jun 2 14:02:14 2021 +0200
>>
>> --dropin for quickly reconfiguring unit tests on the fly
>> ---
>> runall.py | 7 ++++++-
>> tests/test-generators.py | 11 +++++++++++
>> 2 files changed, 17 insertions(+), 1 deletion(-)
>>
>> diff --git a/runall.py b/runall.py
>> index 556eec8..4b735cb 100755
>> --- a/runall.py
>> +++ b/runall.py
>> @@ -20,7 +20,10 @@ if __name__ == '__main__':
>> parser.add_argument('--gtype', dest='gtype', type=str, nargs='+',
>> help="Run only specified generators (medium, cluster, dkim, full, etc)")
>> parser.add_argument('--yamldir', dest='yamldir', type=str, action='store',
>> - help="Load yaml specs from alternate directory")
>> + help="Load yaml specs from alternate directory")
>> + parser.add_argument('--dropin', dest='dropin', type=str, action='store',
>> + help="If set to a specific generator name, replaces its unit test results with the current "
>> + "output in the yaml tests")
>> parser.add_argument('--nomboxo', dest = 'nomboxo', action='store_true',
>> help = 'Skip Mboxo processing')
>> parser.add_argument('--fof', dest='failonfail', action='store_true',
>> @@ -69,6 +72,8 @@ if __name__ == '__main__':
>> if args.gtype and test_type == 'generators':
>> cliargs.append('--generators')
>> cliargs.extend(args.gtype)
>> + if args.dropin:
>> + cliargs.extend(['--dropin', args.dropin])
>> if args.skipnodate and test_type == 'generators':
>> cliargs.append('--skipnodate')
>> rv = subprocess.check_output(cliargs, env=env)
>> diff --git a/tests/test-generators.py b/tests/test-generators.py
>> index aaa3daa..670d732 100755
>> --- a/tests/test-generators.py
>> +++ b/tests/test-generators.py
>> @@ -10,6 +10,7 @@ import yaml
>> import argparse
>> import collections
>> import interfacer
>> +import time
>> import email.utils
>>
>> parse_html = False
>> @@ -150,9 +151,17 @@ def run_tests(args):
>> errors += 1
>> sys.stderr.write("""[FAIL] %s, index %2u: Expected '%s', got '%s'!\n""" %
>> (gen_type, key, expected, actual))
>> + if args.dropin and gen_type == args.dropin:
>> + if expected != actual:
>> + test['generated'] = actual
>> + else:
>> + test['alternate'] = actual
>> else:
>> print("[PASS] %s index %u" % (gen_type, key))
>> mboxfiles = [] # reset for the next set of tests
>> + if args.dropin and errors:
>> + sys.stderr.write("Writing replacement yaml as --dropin was specified\n")
>> + yaml.safe_dump(yml, open(args.load, "w"), sort_keys=False)
>> print("[DONE] %u tests run, %u failed." % (tests_run, errors))
>> if errors:
>> sys.exit(-1)
>> @@ -174,6 +183,8 @@ def main():
>> help="Root directory of Apache Pony Mail")
>> parser.add_argument('--nomboxo', dest = 'nomboxo', action='store_true',
>> help = 'Skip Mboxo processing')
>> + parser.add_argument('--dropin', dest = 'dropin', type=str,
>> + help = 'Perform drop-in replacement of unit test results for the specified generator type [devs only!]')
>> parser.add_argument('--skipnodate', dest = 'skipnodate', action='store_true',
>> help = 'Skip emails with no Date: header (useful for medium generator tests)')
>> args = parser.parse_args()
Re: [incubator-ponymail-unit-tests] 02/05: --dropin for quickly
reconfiguring unit tests on the fly
Posted by sebb <se...@gmail.com>.
-1 to this change.
It is not right to arbitrarily change test data this way.
Changes to test data should only be made if there is a genuine reason to do so
e.g. due to a mistake in the original data or a planned change in the API.
If a test starts failing, the most likely reason is a bug due to a
change in the code, not a bug in the test data.
(There are of course exceptions, e.g. where a test is dependent on
timing or other environmental factors)
On Wed, 2 Jun 2021 at 13:23, <hu...@apache.org> wrote:
>
> This is an automated email from the ASF dual-hosted git repository.
>
> humbedooh pushed a commit to branch master
> in repository https://gitbox.apache.org/repos/asf/incubator-ponymail-unit-tests.git
>
> commit fb8025f06013811eed9e2ea3e4f7946dde1eea16
> Author: Daniel Gruno <hu...@apache.org>
> AuthorDate: Wed Jun 2 14:02:14 2021 +0200
>
> --dropin for quickly reconfiguring unit tests on the fly
> ---
> runall.py | 7 ++++++-
> tests/test-generators.py | 11 +++++++++++
> 2 files changed, 17 insertions(+), 1 deletion(-)
>
> diff --git a/runall.py b/runall.py
> index 556eec8..4b735cb 100755
> --- a/runall.py
> +++ b/runall.py
> @@ -20,7 +20,10 @@ if __name__ == '__main__':
> parser.add_argument('--gtype', dest='gtype', type=str, nargs='+',
> help="Run only specified generators (medium, cluster, dkim, full, etc)")
> parser.add_argument('--yamldir', dest='yamldir', type=str, action='store',
> - help="Load yaml specs from alternate directory")
> + help="Load yaml specs from alternate directory")
> + parser.add_argument('--dropin', dest='dropin', type=str, action='store',
> + help="If set to a specific generator name, replaces its unit test results with the current "
> + "output in the yaml tests")
> parser.add_argument('--nomboxo', dest = 'nomboxo', action='store_true',
> help = 'Skip Mboxo processing')
> parser.add_argument('--fof', dest='failonfail', action='store_true',
> @@ -69,6 +72,8 @@ if __name__ == '__main__':
> if args.gtype and test_type == 'generators':
> cliargs.append('--generators')
> cliargs.extend(args.gtype)
> + if args.dropin:
> + cliargs.extend(['--dropin', args.dropin])
> if args.skipnodate and test_type == 'generators':
> cliargs.append('--skipnodate')
> rv = subprocess.check_output(cliargs, env=env)
> diff --git a/tests/test-generators.py b/tests/test-generators.py
> index aaa3daa..670d732 100755
> --- a/tests/test-generators.py
> +++ b/tests/test-generators.py
> @@ -10,6 +10,7 @@ import yaml
> import argparse
> import collections
> import interfacer
> +import time
> import email.utils
>
> parse_html = False
> @@ -150,9 +151,17 @@ def run_tests(args):
> errors += 1
> sys.stderr.write("""[FAIL] %s, index %2u: Expected '%s', got '%s'!\n""" %
> (gen_type, key, expected, actual))
> + if args.dropin and gen_type == args.dropin:
> + if expected != actual:
> + test['generated'] = actual
> + else:
> + test['alternate'] = actual
> else:
> print("[PASS] %s index %u" % (gen_type, key))
> mboxfiles = [] # reset for the next set of tests
> + if args.dropin and errors:
> + sys.stderr.write("Writing replacement yaml as --dropin was specified\n")
> + yaml.safe_dump(yml, open(args.load, "w"), sort_keys=False)
> print("[DONE] %u tests run, %u failed." % (tests_run, errors))
> if errors:
> sys.exit(-1)
> @@ -174,6 +183,8 @@ def main():
> help="Root directory of Apache Pony Mail")
> parser.add_argument('--nomboxo', dest = 'nomboxo', action='store_true',
> help = 'Skip Mboxo processing')
> + parser.add_argument('--dropin', dest = 'dropin', type=str,
> + help = 'Perform drop-in replacement of unit test results for the specified generator type [devs only!]')
> parser.add_argument('--skipnodate', dest = 'skipnodate', action='store_true',
> help = 'Skip emails with no Date: header (useful for medium generator tests)')
> args = parser.parse_args()