diff options
| -rw-r--r-- | regtests/assets/asm/h6280.asm (renamed from regtests/unidasm/static/h6280.asm) | 0 | ||||
| -rw-r--r-- | regtests/assets/asm/m6502.asm (renamed from regtests/unidasm/static/m6502.asm) | 0 | ||||
| -rw-r--r-- | regtests/assets/asm/z80.asm (renamed from regtests/unidasm/static/z80.asm) | 0 | ||||
| -rw-r--r-- | regtests/assets/bin/h6280.bin (renamed from regtests/unidasm/static/h6280.bin) | bin | 10 -> 10 bytes | |||
| -rw-r--r-- | regtests/assets/bin/m6502.bin (renamed from regtests/unidasm/static/m6502.bin) | bin | 8 -> 8 bytes | |||
| -rw-r--r-- | regtests/assets/bin/z80.bin (renamed from regtests/unidasm/static/z80.bin) | bin | 9 -> 9 bytes | |||
| -rw-r--r-- | regtests/regtests.mak | 11 | ||||
| -rw-r--r-- | regtests/tool_tester/test_tools.py | 31 | ||||
| -rw-r--r-- | regtests/tool_tester/tool_tester/__init__.py | 0 | ||||
| -rw-r--r-- | regtests/tool_tester/tool_tester/_selfexe.py | 97 | ||||
| -rw-r--r-- | regtests/tool_tester/tool_tester/unidasm.py | 72 | ||||
| -rw-r--r-- | regtests/unidasm/unidasm.py | 89 |
12 files changed, 210 insertions, 90 deletions
diff --git a/regtests/unidasm/static/h6280.asm b/regtests/assets/asm/h6280.asm index 73c34eff41a..73c34eff41a 100644 --- a/regtests/unidasm/static/h6280.asm +++ b/regtests/assets/asm/h6280.asm diff --git a/regtests/unidasm/static/m6502.asm b/regtests/assets/asm/m6502.asm index d578baa7df1..d578baa7df1 100644 --- a/regtests/unidasm/static/m6502.asm +++ b/regtests/assets/asm/m6502.asm diff --git a/regtests/unidasm/static/z80.asm b/regtests/assets/asm/z80.asm index e57d2dedf37..e57d2dedf37 100644 --- a/regtests/unidasm/static/z80.asm +++ b/regtests/assets/asm/z80.asm diff --git a/regtests/unidasm/static/h6280.bin b/regtests/assets/bin/h6280.bin Binary files differindex 781727f523f..781727f523f 100644 --- a/regtests/unidasm/static/h6280.bin +++ b/regtests/assets/bin/h6280.bin diff --git a/regtests/unidasm/static/m6502.bin b/regtests/assets/bin/m6502.bin Binary files differindex 853a3d957d3..853a3d957d3 100644 --- a/regtests/unidasm/static/m6502.bin +++ b/regtests/assets/bin/m6502.bin diff --git a/regtests/unidasm/static/z80.bin b/regtests/assets/bin/z80.bin Binary files differindex 824587e3465..824587e3465 100644 --- a/regtests/unidasm/static/z80.bin +++ b/regtests/assets/bin/z80.bin diff --git a/regtests/regtests.mak b/regtests/regtests.mak index 6c76357fadb..bb102ef27d0 100644 --- a/regtests/regtests.mak +++ b/regtests/regtests.mak @@ -43,11 +43,20 @@ #------------------------------------------------- REGTESTS += \ - unidasmtest \ + toolstest \ +# unidasmtest \ # chdmantest \ # jedutiltest \ #------------------------------------------------- +# tools +#------------------------------------------------- + +toolstest: + @echo Running tools unittests + $(PYTHON) regtests/tool_tester/test_tools.py + +#------------------------------------------------- # chdman #------------------------------------------------- diff --git a/regtests/tool_tester/test_tools.py b/regtests/tool_tester/test_tools.py new file mode 100644 index 00000000000..0adfe2da808 --- /dev/null +++ b/regtests/tool_tester/test_tools.py @@ -0,0 +1,31 @@ +#!/usr/bin/python +## +## license:BSD-3-Clause +## copyright-holders:Angelo Salese +## +import sys +from os.path import join, dirname, realpath +import logging +from tool_tester.unidasm import ( + UnidasmTests +) + +if __name__ == "__main__": + # TODO: add colorized messages + # TODO: argparse the logging level + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + + # TODO: for now I'll just use class handlers here to chain test sources + # In an ideal world you want to collect items thru inspect module instead + # https://docs.python.org/3/library/inspect.html + # and isolate by handler name, so that an optional arg can be passed here and launch + # a given test module on user demand + chained_results = [] + # TODO: point to $(regtests)\assets, configure if necessary + assets_folder = join(dirname(dirname(realpath(__file__))), "assets") + for test_cls in [UnidasmTests]: + test_fn = test_cls(assets_folder) + logging.info("Start test suite: %s", test_fn.identifier) + chained_results.append(test_fn.execute_tests(test_fn.compose_tests())) + logging.debug("test results %s", repr(chained_results)) + sys.exit(0 if all(chained_results) else 1) diff --git a/regtests/tool_tester/tool_tester/__init__.py b/regtests/tool_tester/tool_tester/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/regtests/tool_tester/tool_tester/__init__.py diff --git a/regtests/tool_tester/tool_tester/_selfexe.py b/regtests/tool_tester/tool_tester/_selfexe.py new file mode 100644 index 00000000000..a72e3435ef3 --- /dev/null +++ b/regtests/tool_tester/tool_tester/_selfexe.py @@ -0,0 +1,97 @@ +## +## license:BSD-3-Clause +## copyright-holders:Angelo Salese +## +from abc import ABC, abstractmethod +import os +import logging +import subprocess +from typing import Dict, List + +class SelfExeTests(ABC): + def __init__(self, id_exe: str, assets_folder: str): + self.identifier = id_exe + self._exec_path = os.path.join(os.getcwd(), "{0}{1}".format(id_exe, ".exe" if os.name == 'nt' else "")) + self._assets_path = assets_folder + logging.debug("Setup %s executable exe at %s", id_exe, self._exec_path) + + def compose_tests(self) -> Dict: + """Compose a list of tests to be later reused. + + Client can override this method and call super on itself + if it needs finer logging granularity (such as printing with logging.debug) + + Returns: + Dict: key for test_handle, value is an arbitrary list or dataclass driven object (preferably latter) + """ + test_pool = self._collect_tests() + logging.info("Detected %s tests", len(test_pool)) + return test_pool + + @abstractmethod + def _collect_tests(self) -> Dict: + """Mapping fn that the client must override + + Returns: + Dict: key for test_handle, value is an arbitrary list or dataclass driven object (preferably latter) + """ + + def execute_tests(self, test_pool: Dict) -> bool: + """Test executor pipeline + + Args: + test_pool (Dict): a mapping defined by self._collect_tests() + + Returns: + bool: True if all tests passes successfully, False otherwise + """ + test_result = True + for test_handle, test_params in test_pool.items(): + logging.debug("Preparing %s-%s test ", self.identifier, test_handle) + try: + logging.debug("Launching test %s", test_handle) + __launch_fullpath = [self._exec_path] + self._subprocess_args(test_handle, test_params) + logging.debug(__launch_fullpath) + launch_result = subprocess.run( + __launch_fullpath, + encoding="utf-8", + capture_output=True, + check=True, + text=True, + shell=False + ) + logging.debug("Subprocess test %s pass", test_handle) + self._assert_test_result(launch_result, test_handle, test_params) + logging.info("Test %s passed", test_handle) + except (subprocess.CalledProcessError, AssertionError) as ex: + logging.exception(str(ex)) + test_result = False + return test_result + + @abstractmethod + def _subprocess_args(self, test_handle: str, test_params: List) -> List: + """Translator fn for dispatch parameter inputs against a subprocess run + + Args: + test_handle (str): handle identifier + Client can breakdown at will for branching with parameters + test_params (List): parameter List or dataclass object mapper + + Returns: + List: formatted list for subprocess + """ + + @abstractmethod + def _assert_test_result(self, launch_result: subprocess.CompletedProcess, test_handle: str, test_params: List): + """Unit test fn an actual test against the resulting output from subprocess + + A bare minimum is to assert against returncode, stdout and stderr. + If you need to check against binary formats give extra carefulness that the test + is always repeatable and non-OS dependant. + + Args: + launch_result (subprocess.CompletedProcess): result of a given + test_handle (str): handle identifier + Client can breakdown at will for branching with parameters + test_params (List): parameter List or dataclass object mapper + """ diff --git a/regtests/tool_tester/tool_tester/unidasm.py b/regtests/tool_tester/tool_tester/unidasm.py new file mode 100644 index 00000000000..5cbf17cc925 --- /dev/null +++ b/regtests/tool_tester/tool_tester/unidasm.py @@ -0,0 +1,72 @@ +## +## license:BSD-3-Clause +## copyright-holders:Angelo Salese +## +import os +from subprocess import CompletedProcess +from dataclasses import dataclass +import logging +from tool_tester._selfexe import SelfExeTests + +@dataclass(frozen=True) +class UnidasmMapper: + asm_filename: str + bin_filename: str + + def __post_init__(self): + for idx, params in enumerate(zip([self.asm_filename, self.bin_filename], [".asm", ".bin"])): +# assert item.startswith(test_handle), f"{item} {expected_extension} mismatch with arch {target_arch}" + assert params[0].endswith(params[1]), f"{params[0]} mismatch extension {params[1]}" + + +class UnidasmTests(SelfExeTests): + """Unidasm test collector + + For unidasm the strategy is to point at the static folder and + derive unit tests based off what's in static directory itself. + To compose a new test for a given arch: + + 1. craft a new binary file and save it in static subfolder, name it <test_arch>.bin, + where <test_arch> is the name as it is displayed by running unidasm.exe + with no additional params. Keep it simple & short, more sofisticated programs aren't really + the scope of this (we're just testing if the arch disassembles here). + 2. run unidasm $(regtests)/static/<test_arch>.bin -arch <test_arch> > $(regtests)/static/<test_arch>.asm + Note: on windows the resulting asm file newlines must be converted to LF with this + (otherwise you get a diff error later on). + 3. now run make tests or python $(modulepath)/test_tools.py and check if the new test gets captured. + + """ + + def __init__(self, assets_path: str): + super().__init__("unidasm", assets_path) + self._asm_test_folder = os.path.join(assets_path, "asm") + self._bin_test_folder = os.path.join(assets_path, "bin") + logging.debug(self._asm_test_folder) + logging.debug(self._bin_test_folder) + + def _collect_tests(self): + __static_tests = zip( os.listdir(self._asm_test_folder), os.listdir(self._bin_test_folder)) + return { + __asm.split(".")[0]: UnidasmMapper(asm_filename=__asm, bin_filename=__bin) for __asm, __bin in __static_tests + } + + def _subprocess_args(self, test_handle: str, test_params: UnidasmMapper): + return [os.path.join(self._bin_test_folder, test_params.bin_filename), "-arch", test_handle] + + def _assert_test_result(self, launch_result: CompletedProcess, test_handle: str, test_params: UnidasmMapper): + __ret_code = launch_result.returncode + assert __ret_code == 0, f"{test_handle} return code == {__ret_code}" + __actual_stderr = launch_result.stderr + assert not __actual_stderr, f"unexpected stderr {__actual_stderr}" + + with open(os.path.join(self._asm_test_folder, test_params.asm_filename), "r", encoding="utf-8", newline="\n") as txt_h: + __expected_asm = txt_h.read() + __actual_stdout = launch_result.stdout + # TODO: eventually use difflib for error printing + # https://docs.python.org/3/library/difflib.html + assert __actual_stdout == __expected_asm, "{1} test FAILED{0}expected:{0}{2}{0}actual:{0}{3}".format( + os.linesep, + test_handle, + repr(__expected_asm), + repr(__actual_stdout) + ) diff --git a/regtests/unidasm/unidasm.py b/regtests/unidasm/unidasm.py deleted file mode 100644 index 7e9cbc58a76..00000000000 --- a/regtests/unidasm/unidasm.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/python -## -## license:BSD-3-Clause -## copyright-holders:Angelo Salese -## -import os -import sys -import logging -import subprocess -from typing import Callable - -def __assert_valid_filename(filehandle: str, target_arch: str, expected_extension: str): - """Checkout if derived file handle input is the expected one - - Args: - filehandle (str): file handle - target_arch (str): the expected architecture base name - expected_extension (str): the expected extension that the file handle should have - """ - - assert filehandle.startswith(target_arch), f"{filehandle} {expected_extension} mismatch with arch {target_arch}" - assert filehandle.endswith(expected_extension), f"{filehandle} mismatch {expected_extension}" - -def run_process(): - """Process our static directory, derive and execute tests - - For unidasm the strategy is to point at the static folder and - derive unit tests based off what's in static directory itself. - To compose a new test for a given arch: - - 1. craft a new binary file and save it in static subfolder, name it <test_arch>.bin, - where <test_arch> is the name as it is displayed by running unidasm.exe - with no additional params. Keep it simple & short, more sofisticated programs aren't really - the scope of this (we're just testing if the arch disassembles here). - 2. run unidasm $(thisfolder)/static/<test_arch>.bin -arch <test_arch> > $(thisfolder)/static/<test_arch>.asm - On windows the resulting asm file newlines must be converted to LF with this. - 3. now run make tests and check if the new test gets captured. - - """ - unidasm_exe = os.path.join(os.getcwd(), "unidasm{0}".format(".exe" if os.name == 'nt' else "")) - static_tests_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), "static") - static_tests = os.listdir(static_tests_folder) - - test_pool = { - asm_filename.split(".")[0]: [asm_filename, bin_filename] for asm_filename, bin_filename in zip(static_tests[0::2], static_tests[1::2]) - } - - logging.info("Detected %s tests", len(test_pool)) - - for target_arch, params in test_pool.items(): - assert len(params) == 2, f"{target_arch} Unexpected input test structure {params}" - asm_filename, bin_filename = params - __assert_valid_filename(asm_filename, target_arch, ".asm") - __assert_valid_filename(bin_filename, target_arch, ".bin") - - logging.debug("Test arch %s passed internal validation, launching unidasm", target_arch) - - with open(os.path.join(static_tests_folder, asm_filename), "r", encoding="utf-8", newline="\n") as txt_h: - expected_asm = txt_h.read() - - # TODO: checkout how unidasm copes with endianness on 16/32-bit based CPUs - launch_process = subprocess.run( - [unidasm_exe] + [os.path.join(static_tests_folder, bin_filename), "-arch", target_arch], - encoding="utf-8", - capture_output=True, - check=True, - text=True, - shell=False - ) - assert launch_process.returncode == 0, f"{target_arch} return code == {launch_process.returncode}" - # TODO: eventually use difflib - assert launch_process.stdout == expected_asm, "{1} expected:{0}{2}{0}actual:{0}{3}".format( - os.linesep, - target_arch, - repr(expected_asm), - repr(launch_process.stdout) - ) - logging.info("Target arch test %s passed", target_arch) - -if __name__ == "__main__": - logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) - try: - run_process() - except (subprocess.CalledProcessError, AssertionError) as ex: - logging.error(str(ex)) - sys.exit(1) - raise - logging.info("unidasm test successful") - sys.exit(0) |
