summaryrefslogtreecommitdiffstatshomepage
path: root/regtests/tool_tester/tool_tester/_selfexe.py
blob: b5668e20eba3b3194940833617ea4fae01768e59 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
##
## license:BSD-3-Clause
## copyright-holders:Angelo Salese
##
from abc import ABC, abstractmethod
import os
import logging
import subprocess
from typing import Dict, List

class SelfExeTests(ABC):
    identifier: str

    def __init__(self, work_path: str, assets_path: str):
        __id_exe = self.identifier
        self._exec_path = os.path.join(work_path, "{0}{1}".format(__id_exe, ".exe" if os.name == 'nt' else ""))
        self._assets_path = assets_path
        logging.debug("Setup %s executable exe at %s", __id_exe, self._exec_path)

    def compose_tests(self) -> Dict:
        """Compose a list of tests to be later reused.

        Client can override this method and call super on itself
        if it needs finer logging granularity (such as printing with logging.debug)

        Returns:
            Dict: key for test_handle, value is an arbitrary list or dataclass driven object.
        """
        test_pool = self._collect_tests()
        logging.info("Detected %s tests", len(test_pool))
        return test_pool

    @abstractmethod
    def _collect_tests(self) -> Dict:
        """Mapping fn that the client must override

        Returns:
            Dict: key for test_handle, value is an arbitrary list or dataclass driven object.
        """

    def execute_tests(self, test_pool: Dict) -> bool:
        """Test executor pipeline

        Args:
            test_pool (Dict): a mapping defined by self._collect_tests()

        Returns:
            bool: True if all tests passes successfully, False otherwise
        """
        test_result = True
        for test_handle, test_params in test_pool.items():
            __test_name = f"{self.identifier}-{test_handle}"
            logging.debug("Preparing %s test ", __test_name)
            try:
                __launch_fullpath =  [self._exec_path] + self._subprocess_args(test_handle, test_params)
                logging.debug(__launch_fullpath)
                launch_result = subprocess.run(
                    __launch_fullpath, 
                    encoding="utf-8",
                    capture_output=True,
                    # we need to disable check=True here, it would be hairy to handle
                    # for several reasons, namely:
                    # - pngcmp returns a status of 1 when diverging snapshots occurs.
                    # - (eventually) usage instruction test, which technically should return a non-zero status code.
                    #   (which one is tbd)
                    check=False,
                    text=True,
                    shell=False
                )
                logging.debug("Subprocess test %s pass", __test_name)
                self._assert_test_result(launch_result, test_handle, test_params)
                logging.info("Test %s passed", __test_name)
            except (subprocess.CalledProcessError, AssertionError, FileNotFoundError) as ex:
                logging.exception(str(ex)) 
                test_result = False
        return test_result

    @abstractmethod
    def _subprocess_args(self, test_handle: str, test_params: List) -> List:
        """Translator fn for dispatch parameter inputs against a subprocess run

        Args:
            test_handle (str): handle identifier
                Client can breakdown at will for branching with parameters
            test_params (List): parameter List or dataclass object mapper

        Returns:
            List: formatted list for subprocess
        """

    @abstractmethod
    def _assert_test_result(self, launch_result: subprocess.CompletedProcess, test_handle: str, test_params: List):
        """Factory fn for testing out if expected behaviour is satisfied.

        A bare minimum is to assert against returncode, stdout and stderr.
        If you need to check against binary formats give extra carefulness that the test
        is always repeatable and non-OS dependant.

        Args:
            launch_result (subprocess.CompletedProcess): result of a given 
            test_handle (str): handle identifier
                Client can breakdown at will for branching with parameters
            test_params (List): parameter List or dataclass object mapper
        """