summaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
author angelosa <lordkale4@gmail.com>2021-02-15 00:24:03 +0100
committer angelosa <lordkale4@gmail.com>2021-02-15 00:24:03 +0100
commitd944c7e7a9f5836c0405a78ff86cda49936a4a26 (patch)
tree95731a1857baf6442460703a8926c743eb47ca3e
parente9906b995b34d5d30d98f4dadd8edb093e762d05 (diff)
Improve abstract class docspy_regtests
-rw-r--r--regtests/tool_tester/tool_tester/_selfexe.py15
1 files changed, 9 insertions, 6 deletions
diff --git a/regtests/tool_tester/tool_tester/_selfexe.py b/regtests/tool_tester/tool_tester/_selfexe.py
index 7a42ecaa3d5..b5668e20eba 100644
--- a/regtests/tool_tester/tool_tester/_selfexe.py
+++ b/regtests/tool_tester/tool_tester/_selfexe.py
@@ -24,7 +24,7 @@ class SelfExeTests(ABC):
if it needs finer logging granularity (such as printing with logging.debug)
Returns:
- Dict: key for test_handle, value is an arbitrary list or dataclass driven object (preferably latter)
+ Dict: key for test_handle, value is an arbitrary list or dataclass driven object.
"""
test_pool = self._collect_tests()
logging.info("Detected %s tests", len(test_pool))
@@ -35,7 +35,7 @@ class SelfExeTests(ABC):
"""Mapping fn that the client must override
Returns:
- Dict: key for test_handle, value is an arbitrary list or dataclass driven object (preferably latter)
+ Dict: key for test_handle, value is an arbitrary list or dataclass driven object.
"""
def execute_tests(self, test_pool: Dict) -> bool:
@@ -58,8 +58,11 @@ class SelfExeTests(ABC):
__launch_fullpath,
encoding="utf-8",
capture_output=True,
- # TODO: we need to disable check here cause pngcmp has a returncode of 1 when diverging snapshots occurs (wtf)
- #check=True,
+ # we need to disable check=True here, it would be hairy to handle
+ # for several reasons, namely:
+ # - pngcmp returns a status of 1 when diverging snapshots occurs.
+ # - (eventually) usage instruction test, which technically should return a non-zero status code.
+ # (which one is tbd)
check=False,
text=True,
shell=False
@@ -67,7 +70,7 @@ class SelfExeTests(ABC):
logging.debug("Subprocess test %s pass", __test_name)
self._assert_test_result(launch_result, test_handle, test_params)
logging.info("Test %s passed", __test_name)
- except (subprocess.CalledProcessError, AssertionError) as ex:
+ except (subprocess.CalledProcessError, AssertionError, FileNotFoundError) as ex:
logging.exception(str(ex))
test_result = False
return test_result
@@ -87,7 +90,7 @@ class SelfExeTests(ABC):
@abstractmethod
def _assert_test_result(self, launch_result: subprocess.CompletedProcess, test_handle: str, test_params: List):
- """Unit test fn an actual test against the resulting output from subprocess
+ """Factory fn for testing out if expected behaviour is satisfied.
A bare minimum is to assert against returncode, stdout and stderr.
If you need to check against binary formats give extra carefulness that the test