summaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
author angelosa <lordkale4@gmail.com>2021-01-30 04:35:57 +0100
committer angelosa <lordkale4@gmail.com>2021-01-30 04:35:57 +0100
commit13f5a7ed1c277b851e467e8f5c25595d85bbb644 (patch)
treee1dd755511b4e853cd514c26bff1ca049995ac31
parent560db771d58aa8bfc45f0483c083cb3f9c1673b4 (diff)
ABC basic unit testing features, make a dataclass mapper example for test parameters in unidasm, move assets folders to ext folder, refactor trigger to a tool_tester module
-rw-r--r--regtests/assets/asm/h6280.asm (renamed from regtests/unidasm/static/h6280.asm)0
-rw-r--r--regtests/assets/asm/m6502.asm (renamed from regtests/unidasm/static/m6502.asm)0
-rw-r--r--regtests/assets/asm/z80.asm (renamed from regtests/unidasm/static/z80.asm)0
-rw-r--r--regtests/assets/bin/h6280.bin (renamed from regtests/unidasm/static/h6280.bin)bin10 -> 10 bytes
-rw-r--r--regtests/assets/bin/m6502.bin (renamed from regtests/unidasm/static/m6502.bin)bin8 -> 8 bytes
-rw-r--r--regtests/assets/bin/z80.bin (renamed from regtests/unidasm/static/z80.bin)bin9 -> 9 bytes
-rw-r--r--regtests/regtests.mak11
-rw-r--r--regtests/tool_tester/test_tools.py31
-rw-r--r--regtests/tool_tester/tool_tester/__init__.py0
-rw-r--r--regtests/tool_tester/tool_tester/_selfexe.py97
-rw-r--r--regtests/tool_tester/tool_tester/unidasm.py72
-rw-r--r--regtests/unidasm/unidasm.py89
12 files changed, 210 insertions, 90 deletions
diff --git a/regtests/unidasm/static/h6280.asm b/regtests/assets/asm/h6280.asm
index 73c34eff41a..73c34eff41a 100644
--- a/regtests/unidasm/static/h6280.asm
+++ b/regtests/assets/asm/h6280.asm
diff --git a/regtests/unidasm/static/m6502.asm b/regtests/assets/asm/m6502.asm
index d578baa7df1..d578baa7df1 100644
--- a/regtests/unidasm/static/m6502.asm
+++ b/regtests/assets/asm/m6502.asm
diff --git a/regtests/unidasm/static/z80.asm b/regtests/assets/asm/z80.asm
index e57d2dedf37..e57d2dedf37 100644
--- a/regtests/unidasm/static/z80.asm
+++ b/regtests/assets/asm/z80.asm
diff --git a/regtests/unidasm/static/h6280.bin b/regtests/assets/bin/h6280.bin
index 781727f523f..781727f523f 100644
--- a/regtests/unidasm/static/h6280.bin
+++ b/regtests/assets/bin/h6280.bin
Binary files differ
diff --git a/regtests/unidasm/static/m6502.bin b/regtests/assets/bin/m6502.bin
index 853a3d957d3..853a3d957d3 100644
--- a/regtests/unidasm/static/m6502.bin
+++ b/regtests/assets/bin/m6502.bin
Binary files differ
diff --git a/regtests/unidasm/static/z80.bin b/regtests/assets/bin/z80.bin
index 824587e3465..824587e3465 100644
--- a/regtests/unidasm/static/z80.bin
+++ b/regtests/assets/bin/z80.bin
Binary files differ
diff --git a/regtests/regtests.mak b/regtests/regtests.mak
index 6c76357fadb..bb102ef27d0 100644
--- a/regtests/regtests.mak
+++ b/regtests/regtests.mak
@@ -43,11 +43,20 @@
#-------------------------------------------------
REGTESTS += \
- unidasmtest \
+ toolstest \
+# unidasmtest \
# chdmantest \
# jedutiltest \
#-------------------------------------------------
+# tools
+#-------------------------------------------------
+
+toolstest:
+ @echo Running tools unittests
+ $(PYTHON) regtests/tool_tester/test_tools.py
+
+#-------------------------------------------------
# chdman
#-------------------------------------------------
diff --git a/regtests/tool_tester/test_tools.py b/regtests/tool_tester/test_tools.py
new file mode 100644
index 00000000000..0adfe2da808
--- /dev/null
+++ b/regtests/tool_tester/test_tools.py
@@ -0,0 +1,31 @@
+#!/usr/bin/python
+##
+## license:BSD-3-Clause
+## copyright-holders:Angelo Salese
+##
+import sys
+from os.path import join, dirname, realpath
+import logging
+from tool_tester.unidasm import (
+ UnidasmTests
+)
+
+if __name__ == "__main__":
+ # TODO: add colorized messages
+ # TODO: argparse the logging level
+ logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
+
+ # TODO: for now I'll just use class handlers here to chain test sources
+ # In an ideal world you want to collect items thru inspect module instead
+ # https://docs.python.org/3/library/inspect.html
+ # and isolate by handler name, so that an optional arg can be passed here and launch
+ # a given test module on user demand
+ chained_results = []
+ # TODO: point to $(regtests)\assets, configure if necessary
+ assets_folder = join(dirname(dirname(realpath(__file__))), "assets")
+ for test_cls in [UnidasmTests]:
+ test_fn = test_cls(assets_folder)
+ logging.info("Start test suite: %s", test_fn.identifier)
+ chained_results.append(test_fn.execute_tests(test_fn.compose_tests()))
+ logging.debug("test results %s", repr(chained_results))
+ sys.exit(0 if all(chained_results) else 1)
diff --git a/regtests/tool_tester/tool_tester/__init__.py b/regtests/tool_tester/tool_tester/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/regtests/tool_tester/tool_tester/__init__.py
diff --git a/regtests/tool_tester/tool_tester/_selfexe.py b/regtests/tool_tester/tool_tester/_selfexe.py
new file mode 100644
index 00000000000..a72e3435ef3
--- /dev/null
+++ b/regtests/tool_tester/tool_tester/_selfexe.py
@@ -0,0 +1,97 @@
+##
+## license:BSD-3-Clause
+## copyright-holders:Angelo Salese
+##
+from abc import ABC, abstractmethod
+import os
+import logging
+import subprocess
+from typing import Dict, List
+
+class SelfExeTests(ABC):
+ def __init__(self, id_exe: str, assets_folder: str):
+ self.identifier = id_exe
+ self._exec_path = os.path.join(os.getcwd(), "{0}{1}".format(id_exe, ".exe" if os.name == 'nt' else ""))
+ self._assets_path = assets_folder
+ logging.debug("Setup %s executable exe at %s", id_exe, self._exec_path)
+
+ def compose_tests(self) -> Dict:
+ """Compose a list of tests to be later reused.
+
+ Client can override this method and call super on itself
+ if it needs finer logging granularity (such as printing with logging.debug)
+
+ Returns:
+ Dict: key for test_handle, value is an arbitrary list or dataclass driven object (preferably latter)
+ """
+ test_pool = self._collect_tests()
+ logging.info("Detected %s tests", len(test_pool))
+ return test_pool
+
+ @abstractmethod
+ def _collect_tests(self) -> Dict:
+ """Mapping fn that the client must override
+
+ Returns:
+ Dict: key for test_handle, value is an arbitrary list or dataclass driven object (preferably latter)
+ """
+
+ def execute_tests(self, test_pool: Dict) -> bool:
+ """Test executor pipeline
+
+ Args:
+ test_pool (Dict): a mapping defined by self._collect_tests()
+
+ Returns:
+ bool: True if all tests passes successfully, False otherwise
+ """
+ test_result = True
+ for test_handle, test_params in test_pool.items():
+ logging.debug("Preparing %s-%s test ", self.identifier, test_handle)
+ try:
+ logging.debug("Launching test %s", test_handle)
+ __launch_fullpath = [self._exec_path] + self._subprocess_args(test_handle, test_params)
+ logging.debug(__launch_fullpath)
+ launch_result = subprocess.run(
+ __launch_fullpath,
+ encoding="utf-8",
+ capture_output=True,
+ check=True,
+ text=True,
+ shell=False
+ )
+ logging.debug("Subprocess test %s pass", test_handle)
+ self._assert_test_result(launch_result, test_handle, test_params)
+ logging.info("Test %s passed", test_handle)
+ except (subprocess.CalledProcessError, AssertionError) as ex:
+ logging.exception(str(ex))
+ test_result = False
+ return test_result
+
+ @abstractmethod
+ def _subprocess_args(self, test_handle: str, test_params: List) -> List:
+ """Translator fn for dispatch parameter inputs against a subprocess run
+
+ Args:
+ test_handle (str): handle identifier
+ Client can breakdown at will for branching with parameters
+ test_params (List): parameter List or dataclass object mapper
+
+ Returns:
+ List: formatted list for subprocess
+ """
+
+ @abstractmethod
+ def _assert_test_result(self, launch_result: subprocess.CompletedProcess, test_handle: str, test_params: List):
+ """Unit test fn an actual test against the resulting output from subprocess
+
+ A bare minimum is to assert against returncode, stdout and stderr.
+ If you need to check against binary formats give extra carefulness that the test
+ is always repeatable and non-OS dependant.
+
+ Args:
+ launch_result (subprocess.CompletedProcess): result of a given
+ test_handle (str): handle identifier
+ Client can breakdown at will for branching with parameters
+ test_params (List): parameter List or dataclass object mapper
+ """
diff --git a/regtests/tool_tester/tool_tester/unidasm.py b/regtests/tool_tester/tool_tester/unidasm.py
new file mode 100644
index 00000000000..5cbf17cc925
--- /dev/null
+++ b/regtests/tool_tester/tool_tester/unidasm.py
@@ -0,0 +1,72 @@
+##
+## license:BSD-3-Clause
+## copyright-holders:Angelo Salese
+##
+import os
+from subprocess import CompletedProcess
+from dataclasses import dataclass
+import logging
+from tool_tester._selfexe import SelfExeTests
+
+@dataclass(frozen=True)
+class UnidasmMapper:
+ asm_filename: str
+ bin_filename: str
+
+ def __post_init__(self):
+ for idx, params in enumerate(zip([self.asm_filename, self.bin_filename], [".asm", ".bin"])):
+# assert item.startswith(test_handle), f"{item} {expected_extension} mismatch with arch {target_arch}"
+ assert params[0].endswith(params[1]), f"{params[0]} mismatch extension {params[1]}"
+
+
+class UnidasmTests(SelfExeTests):
+ """Unidasm test collector
+
+ For unidasm the strategy is to point at the static folder and
+ derive unit tests based off what's in static directory itself.
+ To compose a new test for a given arch:
+
+ 1. craft a new binary file and save it in static subfolder, name it <test_arch>.bin,
+ where <test_arch> is the name as it is displayed by running unidasm.exe
+ with no additional params. Keep it simple & short, more sofisticated programs aren't really
+ the scope of this (we're just testing if the arch disassembles here).
+ 2. run unidasm $(regtests)/static/<test_arch>.bin -arch <test_arch> > $(regtests)/static/<test_arch>.asm
+ Note: on windows the resulting asm file newlines must be converted to LF with this
+ (otherwise you get a diff error later on).
+ 3. now run make tests or python $(modulepath)/test_tools.py and check if the new test gets captured.
+
+ """
+
+ def __init__(self, assets_path: str):
+ super().__init__("unidasm", assets_path)
+ self._asm_test_folder = os.path.join(assets_path, "asm")
+ self._bin_test_folder = os.path.join(assets_path, "bin")
+ logging.debug(self._asm_test_folder)
+ logging.debug(self._bin_test_folder)
+
+ def _collect_tests(self):
+ __static_tests = zip( os.listdir(self._asm_test_folder), os.listdir(self._bin_test_folder))
+ return {
+ __asm.split(".")[0]: UnidasmMapper(asm_filename=__asm, bin_filename=__bin) for __asm, __bin in __static_tests
+ }
+
+ def _subprocess_args(self, test_handle: str, test_params: UnidasmMapper):
+ return [os.path.join(self._bin_test_folder, test_params.bin_filename), "-arch", test_handle]
+
+ def _assert_test_result(self, launch_result: CompletedProcess, test_handle: str, test_params: UnidasmMapper):
+ __ret_code = launch_result.returncode
+ assert __ret_code == 0, f"{test_handle} return code == {__ret_code}"
+ __actual_stderr = launch_result.stderr
+ assert not __actual_stderr, f"unexpected stderr {__actual_stderr}"
+
+ with open(os.path.join(self._asm_test_folder, test_params.asm_filename), "r", encoding="utf-8", newline="\n") as txt_h:
+ __expected_asm = txt_h.read()
+ __actual_stdout = launch_result.stdout
+ # TODO: eventually use difflib for error printing
+ # https://docs.python.org/3/library/difflib.html
+ assert __actual_stdout == __expected_asm, "{1} test FAILED{0}expected:{0}{2}{0}actual:{0}{3}".format(
+ os.linesep,
+ test_handle,
+ repr(__expected_asm),
+ repr(__actual_stdout)
+ )
diff --git a/regtests/unidasm/unidasm.py b/regtests/unidasm/unidasm.py
deleted file mode 100644
index 7e9cbc58a76..00000000000
--- a/regtests/unidasm/unidasm.py
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/usr/bin/python
-##
-## license:BSD-3-Clause
-## copyright-holders:Angelo Salese
-##
-import os
-import sys
-import logging
-import subprocess
-from typing import Callable
-
-def __assert_valid_filename(filehandle: str, target_arch: str, expected_extension: str):
- """Checkout if derived file handle input is the expected one
-
- Args:
- filehandle (str): file handle
- target_arch (str): the expected architecture base name
- expected_extension (str): the expected extension that the file handle should have
- """
-
- assert filehandle.startswith(target_arch), f"{filehandle} {expected_extension} mismatch with arch {target_arch}"
- assert filehandle.endswith(expected_extension), f"{filehandle} mismatch {expected_extension}"
-
-def run_process():
- """Process our static directory, derive and execute tests
-
- For unidasm the strategy is to point at the static folder and
- derive unit tests based off what's in static directory itself.
- To compose a new test for a given arch:
-
- 1. craft a new binary file and save it in static subfolder, name it <test_arch>.bin,
- where <test_arch> is the name as it is displayed by running unidasm.exe
- with no additional params. Keep it simple & short, more sofisticated programs aren't really
- the scope of this (we're just testing if the arch disassembles here).
- 2. run unidasm $(thisfolder)/static/<test_arch>.bin -arch <test_arch> > $(thisfolder)/static/<test_arch>.asm
- On windows the resulting asm file newlines must be converted to LF with this.
- 3. now run make tests and check if the new test gets captured.
-
- """
- unidasm_exe = os.path.join(os.getcwd(), "unidasm{0}".format(".exe" if os.name == 'nt' else ""))
- static_tests_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), "static")
- static_tests = os.listdir(static_tests_folder)
-
- test_pool = {
- asm_filename.split(".")[0]: [asm_filename, bin_filename] for asm_filename, bin_filename in zip(static_tests[0::2], static_tests[1::2])
- }
-
- logging.info("Detected %s tests", len(test_pool))
-
- for target_arch, params in test_pool.items():
- assert len(params) == 2, f"{target_arch} Unexpected input test structure {params}"
- asm_filename, bin_filename = params
- __assert_valid_filename(asm_filename, target_arch, ".asm")
- __assert_valid_filename(bin_filename, target_arch, ".bin")
-
- logging.debug("Test arch %s passed internal validation, launching unidasm", target_arch)
-
- with open(os.path.join(static_tests_folder, asm_filename), "r", encoding="utf-8", newline="\n") as txt_h:
- expected_asm = txt_h.read()
-
- # TODO: checkout how unidasm copes with endianness on 16/32-bit based CPUs
- launch_process = subprocess.run(
- [unidasm_exe] + [os.path.join(static_tests_folder, bin_filename), "-arch", target_arch],
- encoding="utf-8",
- capture_output=True,
- check=True,
- text=True,
- shell=False
- )
- assert launch_process.returncode == 0, f"{target_arch} return code == {launch_process.returncode}"
- # TODO: eventually use difflib
- assert launch_process.stdout == expected_asm, "{1} expected:{0}{2}{0}actual:{0}{3}".format(
- os.linesep,
- target_arch,
- repr(expected_asm),
- repr(launch_process.stdout)
- )
- logging.info("Target arch test %s passed", target_arch)
-
-if __name__ == "__main__":
- logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
- try:
- run_process()
- except (subprocess.CalledProcessError, AssertionError) as ex:
- logging.error(str(ex))
- sys.exit(1)
- raise
- logging.info("unidasm test successful")
- sys.exit(0)