From 13f5a7ed1c277b851e467e8f5c25595d85bbb644 Mon Sep 17 00:00:00 2001 From: angelosa Date: Sat, 30 Jan 2021 04:35:57 +0100 Subject: ABC basic unit testing features, make a dataclass mapper example for test parameters in unidasm, move assets folders to ext folder, refactor trigger to a tool_tester module --- regtests/assets/asm/h6280.asm | 6 ++ regtests/assets/asm/m6502.asm | 5 ++ regtests/assets/asm/z80.asm | 4 ++ regtests/assets/bin/h6280.bin | Bin 0 -> 10 bytes regtests/assets/bin/m6502.bin | Bin 0 -> 8 bytes regtests/assets/bin/z80.bin | Bin 0 -> 9 bytes regtests/regtests.mak | 11 ++- regtests/tool_tester/test_tools.py | 31 +++++++++ regtests/tool_tester/tool_tester/__init__.py | 0 regtests/tool_tester/tool_tester/_selfexe.py | 97 +++++++++++++++++++++++++++ regtests/tool_tester/tool_tester/unidasm.py | 72 ++++++++++++++++++++ regtests/unidasm/static/h6280.asm | 6 -- regtests/unidasm/static/h6280.bin | Bin 10 -> 0 bytes regtests/unidasm/static/m6502.asm | 5 -- regtests/unidasm/static/m6502.bin | Bin 8 -> 0 bytes regtests/unidasm/static/z80.asm | 4 -- regtests/unidasm/static/z80.bin | Bin 9 -> 0 bytes regtests/unidasm/unidasm.py | 89 ------------------------ 18 files changed, 225 insertions(+), 105 deletions(-) create mode 100644 regtests/assets/asm/h6280.asm create mode 100644 regtests/assets/asm/m6502.asm create mode 100644 regtests/assets/asm/z80.asm create mode 100644 regtests/assets/bin/h6280.bin create mode 100644 regtests/assets/bin/m6502.bin create mode 100644 regtests/assets/bin/z80.bin create mode 100644 regtests/tool_tester/test_tools.py create mode 100644 regtests/tool_tester/tool_tester/__init__.py create mode 100644 regtests/tool_tester/tool_tester/_selfexe.py create mode 100644 regtests/tool_tester/tool_tester/unidasm.py delete mode 100644 regtests/unidasm/static/h6280.asm delete mode 100644 regtests/unidasm/static/h6280.bin delete mode 100644 regtests/unidasm/static/m6502.asm delete mode 100644 regtests/unidasm/static/m6502.bin delete mode 100644 regtests/unidasm/static/z80.asm delete mode 100644 regtests/unidasm/static/z80.bin delete mode 100644 regtests/unidasm/unidasm.py diff --git a/regtests/assets/asm/h6280.asm b/regtests/assets/asm/h6280.asm new file mode 100644 index 00000000000..73c34eff41a --- /dev/null +++ b/regtests/assets/asm/h6280.asm @@ -0,0 +1,6 @@ +0: 78 sei +1: d4 csh +2: d8 cld +3: a9 ff lda #$FF +5: 53 ff tam #$FF +7: 4c 00 e0 jmp $E000 diff --git a/regtests/assets/asm/m6502.asm b/regtests/assets/asm/m6502.asm new file mode 100644 index 00000000000..d578baa7df1 --- /dev/null +++ b/regtests/assets/asm/m6502.asm @@ -0,0 +1,5 @@ +0: 78 sei +1: d8 cld +2: a2 ff ldx #$ff +4: 9a txs +5: 4c 00 e0 jmp $e000 diff --git a/regtests/assets/asm/z80.asm b/regtests/assets/asm/z80.asm new file mode 100644 index 00000000000..e57d2dedf37 --- /dev/null +++ b/regtests/assets/asm/z80.asm @@ -0,0 +1,4 @@ +0: f3 di +1: ed 56 im 1 +3: 31 00 00 ld sp,$0000 +6: c3 00 00 jp $0000 diff --git a/regtests/assets/bin/h6280.bin b/regtests/assets/bin/h6280.bin new file mode 100644 index 00000000000..781727f523f Binary files /dev/null and b/regtests/assets/bin/h6280.bin differ diff --git a/regtests/assets/bin/m6502.bin b/regtests/assets/bin/m6502.bin new file mode 100644 index 00000000000..853a3d957d3 Binary files /dev/null and b/regtests/assets/bin/m6502.bin differ diff --git a/regtests/assets/bin/z80.bin b/regtests/assets/bin/z80.bin new file mode 100644 index 00000000000..824587e3465 Binary files /dev/null and b/regtests/assets/bin/z80.bin differ diff --git a/regtests/regtests.mak b/regtests/regtests.mak index 6c76357fadb..bb102ef27d0 100644 --- a/regtests/regtests.mak +++ b/regtests/regtests.mak @@ -43,10 +43,19 @@ #------------------------------------------------- REGTESTS += \ - unidasmtest \ + toolstest \ +# unidasmtest \ # chdmantest \ # jedutiltest \ +#------------------------------------------------- +# tools +#------------------------------------------------- + +toolstest: + @echo Running tools unittests + $(PYTHON) regtests/tool_tester/test_tools.py + #------------------------------------------------- # chdman #------------------------------------------------- diff --git a/regtests/tool_tester/test_tools.py b/regtests/tool_tester/test_tools.py new file mode 100644 index 00000000000..0adfe2da808 --- /dev/null +++ b/regtests/tool_tester/test_tools.py @@ -0,0 +1,31 @@ +#!/usr/bin/python +## +## license:BSD-3-Clause +## copyright-holders:Angelo Salese +## +import sys +from os.path import join, dirname, realpath +import logging +from tool_tester.unidasm import ( + UnidasmTests +) + +if __name__ == "__main__": + # TODO: add colorized messages + # TODO: argparse the logging level + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + + # TODO: for now I'll just use class handlers here to chain test sources + # In an ideal world you want to collect items thru inspect module instead + # https://docs.python.org/3/library/inspect.html + # and isolate by handler name, so that an optional arg can be passed here and launch + # a given test module on user demand + chained_results = [] + # TODO: point to $(regtests)\assets, configure if necessary + assets_folder = join(dirname(dirname(realpath(__file__))), "assets") + for test_cls in [UnidasmTests]: + test_fn = test_cls(assets_folder) + logging.info("Start test suite: %s", test_fn.identifier) + chained_results.append(test_fn.execute_tests(test_fn.compose_tests())) + logging.debug("test results %s", repr(chained_results)) + sys.exit(0 if all(chained_results) else 1) diff --git a/regtests/tool_tester/tool_tester/__init__.py b/regtests/tool_tester/tool_tester/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/regtests/tool_tester/tool_tester/_selfexe.py b/regtests/tool_tester/tool_tester/_selfexe.py new file mode 100644 index 00000000000..a72e3435ef3 --- /dev/null +++ b/regtests/tool_tester/tool_tester/_selfexe.py @@ -0,0 +1,97 @@ +## +## license:BSD-3-Clause +## copyright-holders:Angelo Salese +## +from abc import ABC, abstractmethod +import os +import logging +import subprocess +from typing import Dict, List + +class SelfExeTests(ABC): + def __init__(self, id_exe: str, assets_folder: str): + self.identifier = id_exe + self._exec_path = os.path.join(os.getcwd(), "{0}{1}".format(id_exe, ".exe" if os.name == 'nt' else "")) + self._assets_path = assets_folder + logging.debug("Setup %s executable exe at %s", id_exe, self._exec_path) + + def compose_tests(self) -> Dict: + """Compose a list of tests to be later reused. + + Client can override this method and call super on itself + if it needs finer logging granularity (such as printing with logging.debug) + + Returns: + Dict: key for test_handle, value is an arbitrary list or dataclass driven object (preferably latter) + """ + test_pool = self._collect_tests() + logging.info("Detected %s tests", len(test_pool)) + return test_pool + + @abstractmethod + def _collect_tests(self) -> Dict: + """Mapping fn that the client must override + + Returns: + Dict: key for test_handle, value is an arbitrary list or dataclass driven object (preferably latter) + """ + + def execute_tests(self, test_pool: Dict) -> bool: + """Test executor pipeline + + Args: + test_pool (Dict): a mapping defined by self._collect_tests() + + Returns: + bool: True if all tests passes successfully, False otherwise + """ + test_result = True + for test_handle, test_params in test_pool.items(): + logging.debug("Preparing %s-%s test ", self.identifier, test_handle) + try: + logging.debug("Launching test %s", test_handle) + __launch_fullpath = [self._exec_path] + self._subprocess_args(test_handle, test_params) + logging.debug(__launch_fullpath) + launch_result = subprocess.run( + __launch_fullpath, + encoding="utf-8", + capture_output=True, + check=True, + text=True, + shell=False + ) + logging.debug("Subprocess test %s pass", test_handle) + self._assert_test_result(launch_result, test_handle, test_params) + logging.info("Test %s passed", test_handle) + except (subprocess.CalledProcessError, AssertionError) as ex: + logging.exception(str(ex)) + test_result = False + return test_result + + @abstractmethod + def _subprocess_args(self, test_handle: str, test_params: List) -> List: + """Translator fn for dispatch parameter inputs against a subprocess run + + Args: + test_handle (str): handle identifier + Client can breakdown at will for branching with parameters + test_params (List): parameter List or dataclass object mapper + + Returns: + List: formatted list for subprocess + """ + + @abstractmethod + def _assert_test_result(self, launch_result: subprocess.CompletedProcess, test_handle: str, test_params: List): + """Unit test fn an actual test against the resulting output from subprocess + + A bare minimum is to assert against returncode, stdout and stderr. + If you need to check against binary formats give extra carefulness that the test + is always repeatable and non-OS dependant. + + Args: + launch_result (subprocess.CompletedProcess): result of a given + test_handle (str): handle identifier + Client can breakdown at will for branching with parameters + test_params (List): parameter List or dataclass object mapper + """ diff --git a/regtests/tool_tester/tool_tester/unidasm.py b/regtests/tool_tester/tool_tester/unidasm.py new file mode 100644 index 00000000000..5cbf17cc925 --- /dev/null +++ b/regtests/tool_tester/tool_tester/unidasm.py @@ -0,0 +1,72 @@ +## +## license:BSD-3-Clause +## copyright-holders:Angelo Salese +## +import os +from subprocess import CompletedProcess +from dataclasses import dataclass +import logging +from tool_tester._selfexe import SelfExeTests + +@dataclass(frozen=True) +class UnidasmMapper: + asm_filename: str + bin_filename: str + + def __post_init__(self): + for idx, params in enumerate(zip([self.asm_filename, self.bin_filename], [".asm", ".bin"])): +# assert item.startswith(test_handle), f"{item} {expected_extension} mismatch with arch {target_arch}" + assert params[0].endswith(params[1]), f"{params[0]} mismatch extension {params[1]}" + + +class UnidasmTests(SelfExeTests): + """Unidasm test collector + + For unidasm the strategy is to point at the static folder and + derive unit tests based off what's in static directory itself. + To compose a new test for a given arch: + + 1. craft a new binary file and save it in static subfolder, name it .bin, + where is the name as it is displayed by running unidasm.exe + with no additional params. Keep it simple & short, more sofisticated programs aren't really + the scope of this (we're just testing if the arch disassembles here). + 2. run unidasm $(regtests)/static/.bin -arch > $(regtests)/static/.asm + Note: on windows the resulting asm file newlines must be converted to LF with this + (otherwise you get a diff error later on). + 3. now run make tests or python $(modulepath)/test_tools.py and check if the new test gets captured. + + """ + + def __init__(self, assets_path: str): + super().__init__("unidasm", assets_path) + self._asm_test_folder = os.path.join(assets_path, "asm") + self._bin_test_folder = os.path.join(assets_path, "bin") + logging.debug(self._asm_test_folder) + logging.debug(self._bin_test_folder) + + def _collect_tests(self): + __static_tests = zip( os.listdir(self._asm_test_folder), os.listdir(self._bin_test_folder)) + return { + __asm.split(".")[0]: UnidasmMapper(asm_filename=__asm, bin_filename=__bin) for __asm, __bin in __static_tests + } + + def _subprocess_args(self, test_handle: str, test_params: UnidasmMapper): + return [os.path.join(self._bin_test_folder, test_params.bin_filename), "-arch", test_handle] + + def _assert_test_result(self, launch_result: CompletedProcess, test_handle: str, test_params: UnidasmMapper): + __ret_code = launch_result.returncode + assert __ret_code == 0, f"{test_handle} return code == {__ret_code}" + __actual_stderr = launch_result.stderr + assert not __actual_stderr, f"unexpected stderr {__actual_stderr}" + + with open(os.path.join(self._asm_test_folder, test_params.asm_filename), "r", encoding="utf-8", newline="\n") as txt_h: + __expected_asm = txt_h.read() + __actual_stdout = launch_result.stdout + # TODO: eventually use difflib for error printing + # https://docs.python.org/3/library/difflib.html + assert __actual_stdout == __expected_asm, "{1} test FAILED{0}expected:{0}{2}{0}actual:{0}{3}".format( + os.linesep, + test_handle, + repr(__expected_asm), + repr(__actual_stdout) + ) diff --git a/regtests/unidasm/static/h6280.asm b/regtests/unidasm/static/h6280.asm deleted file mode 100644 index 73c34eff41a..00000000000 --- a/regtests/unidasm/static/h6280.asm +++ /dev/null @@ -1,6 +0,0 @@ -0: 78 sei -1: d4 csh -2: d8 cld -3: a9 ff lda #$FF -5: 53 ff tam #$FF -7: 4c 00 e0 jmp $E000 diff --git a/regtests/unidasm/static/h6280.bin b/regtests/unidasm/static/h6280.bin deleted file mode 100644 index 781727f523f..00000000000 Binary files a/regtests/unidasm/static/h6280.bin and /dev/null differ diff --git a/regtests/unidasm/static/m6502.asm b/regtests/unidasm/static/m6502.asm deleted file mode 100644 index d578baa7df1..00000000000 --- a/regtests/unidasm/static/m6502.asm +++ /dev/null @@ -1,5 +0,0 @@ -0: 78 sei -1: d8 cld -2: a2 ff ldx #$ff -4: 9a txs -5: 4c 00 e0 jmp $e000 diff --git a/regtests/unidasm/static/m6502.bin b/regtests/unidasm/static/m6502.bin deleted file mode 100644 index 853a3d957d3..00000000000 Binary files a/regtests/unidasm/static/m6502.bin and /dev/null differ diff --git a/regtests/unidasm/static/z80.asm b/regtests/unidasm/static/z80.asm deleted file mode 100644 index e57d2dedf37..00000000000 --- a/regtests/unidasm/static/z80.asm +++ /dev/null @@ -1,4 +0,0 @@ -0: f3 di -1: ed 56 im 1 -3: 31 00 00 ld sp,$0000 -6: c3 00 00 jp $0000 diff --git a/regtests/unidasm/static/z80.bin b/regtests/unidasm/static/z80.bin deleted file mode 100644 index 824587e3465..00000000000 Binary files a/regtests/unidasm/static/z80.bin and /dev/null differ diff --git a/regtests/unidasm/unidasm.py b/regtests/unidasm/unidasm.py deleted file mode 100644 index 7e9cbc58a76..00000000000 --- a/regtests/unidasm/unidasm.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/python -## -## license:BSD-3-Clause -## copyright-holders:Angelo Salese -## -import os -import sys -import logging -import subprocess -from typing import Callable - -def __assert_valid_filename(filehandle: str, target_arch: str, expected_extension: str): - """Checkout if derived file handle input is the expected one - - Args: - filehandle (str): file handle - target_arch (str): the expected architecture base name - expected_extension (str): the expected extension that the file handle should have - """ - - assert filehandle.startswith(target_arch), f"{filehandle} {expected_extension} mismatch with arch {target_arch}" - assert filehandle.endswith(expected_extension), f"{filehandle} mismatch {expected_extension}" - -def run_process(): - """Process our static directory, derive and execute tests - - For unidasm the strategy is to point at the static folder and - derive unit tests based off what's in static directory itself. - To compose a new test for a given arch: - - 1. craft a new binary file and save it in static subfolder, name it .bin, - where is the name as it is displayed by running unidasm.exe - with no additional params. Keep it simple & short, more sofisticated programs aren't really - the scope of this (we're just testing if the arch disassembles here). - 2. run unidasm $(thisfolder)/static/.bin -arch > $(thisfolder)/static/.asm - On windows the resulting asm file newlines must be converted to LF with this. - 3. now run make tests and check if the new test gets captured. - - """ - unidasm_exe = os.path.join(os.getcwd(), "unidasm{0}".format(".exe" if os.name == 'nt' else "")) - static_tests_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), "static") - static_tests = os.listdir(static_tests_folder) - - test_pool = { - asm_filename.split(".")[0]: [asm_filename, bin_filename] for asm_filename, bin_filename in zip(static_tests[0::2], static_tests[1::2]) - } - - logging.info("Detected %s tests", len(test_pool)) - - for target_arch, params in test_pool.items(): - assert len(params) == 2, f"{target_arch} Unexpected input test structure {params}" - asm_filename, bin_filename = params - __assert_valid_filename(asm_filename, target_arch, ".asm") - __assert_valid_filename(bin_filename, target_arch, ".bin") - - logging.debug("Test arch %s passed internal validation, launching unidasm", target_arch) - - with open(os.path.join(static_tests_folder, asm_filename), "r", encoding="utf-8", newline="\n") as txt_h: - expected_asm = txt_h.read() - - # TODO: checkout how unidasm copes with endianness on 16/32-bit based CPUs - launch_process = subprocess.run( - [unidasm_exe] + [os.path.join(static_tests_folder, bin_filename), "-arch", target_arch], - encoding="utf-8", - capture_output=True, - check=True, - text=True, - shell=False - ) - assert launch_process.returncode == 0, f"{target_arch} return code == {launch_process.returncode}" - # TODO: eventually use difflib - assert launch_process.stdout == expected_asm, "{1} expected:{0}{2}{0}actual:{0}{3}".format( - os.linesep, - target_arch, - repr(expected_asm), - repr(launch_process.stdout) - ) - logging.info("Target arch test %s passed", target_arch) - -if __name__ == "__main__": - logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) - try: - run_process() - except (subprocess.CalledProcessError, AssertionError) as ex: - logging.error(str(ex)) - sys.exit(1) - raise - logging.info("unidasm test successful") - sys.exit(0) -- cgit v1.2.3