From d394b2d24936842a67c913f2bcc6c9af74003b74 Mon Sep 17 00:00:00 2001 From: Nicolas Kruse Date: Mon, 12 Jan 2026 16:57:54 +0100 Subject: [PATCH 01/27] add_read_value_remote backend function renamed and docstring updated --- src/copapy/_compiler.py | 1 + src/copapy/_target.py | 12 ++++++++++-- src/copapy/backend.py | 4 ++-- tests/test_branching_stencils.py | 7 +++---- tests/test_compile.py | 7 +++---- tests/test_compile_aarch64.py | 4 ++-- tests/test_compile_armv7.py | 4 ++-- tests/test_compile_div.py | 8 ++++---- tests/test_compile_math.py | 17 ++++++++--------- tests/test_ops_aarch64.py | 4 ++-- tests/test_ops_armv6.py | 4 ++-- tests/test_ops_armv7.py | 4 ++-- tests/test_ops_x86.py | 8 ++++---- 13 files changed, 45 insertions(+), 39 deletions(-) diff --git a/src/copapy/_compiler.py b/src/copapy/_compiler.py index 9a17f8c..cbade28 100644 --- a/src/copapy/_compiler.py +++ b/src/copapy/_compiler.py @@ -7,6 +7,7 @@ from ._basic_types import Net, Node, Store, CPConstant, Op, transl_type def stable_toposort(edges: Iterable[tuple[Node, Node]]) -> list[Node]: """Perform a stable topological sort on a directed acyclic graph (DAG). + Arguments: edges: Iterable of (u, v) pairs meaning u -> v diff --git a/src/copapy/_target.py b/src/copapy/_target.py index 575c013..f8a64b0 100644 --- a/src/copapy/_target.py +++ b/src/copapy/_target.py @@ -13,7 +13,15 @@ TRet = TypeVar("TRet", Iterable[int | float], int, float) _jit_cache: dict[Any, tuple['Target', tuple[value[Any] | Iterable[value[Any]], ...], NumLike | Iterable[NumLike]]] = {} -def add_read_command(dw: binw.data_writer, variables: dict[Net, tuple[int, int, str]], net: Net) -> None: +def add_read_value_remote(dw: binw.data_writer, variables: dict[Net, tuple[int, int, str]], net: Net) -> None: + """Adds a read memory to stdout command to the data_writer dw. + + Arguments: + dw: data_writer to add the command to + variables: A dict for looking up variables by Net. The value is a tuple. + of relative address in memory, size in bytes and data type. + net: Variable specified by Net to read from memory and write to stdout. + """ assert net in variables, f"Variable {net} not found in data writer variables" addr, lengths, _ = variables[net] dw.write_com(binw.Command.READ_DATA) @@ -188,5 +196,5 @@ class Target(): def read_value_remote(self, variable: value[Any]) -> None: """Reads the raw data of a value by the runner.""" dw = binw.data_writer(self.sdb.byteorder) - add_read_command(dw, self._values, variable.net) + add_read_value_remote(dw, self._values, variable.net) assert coparun(self._context, dw.get_data()) > 0 diff --git a/src/copapy/backend.py b/src/copapy/backend.py index 1593c57..057f306 100644 --- a/src/copapy/backend.py +++ b/src/copapy/backend.py @@ -3,14 +3,14 @@ Backend module for Copapy: contains internal data types and give access to compiler internals and debugging tools. """ -from ._target import add_read_command +from ._target import add_read_value_remote from ._basic_types import Net, Op, Node, CPConstant, Store, stencil_db_from_package from ._compiler import compile_to_dag, \ stable_toposort, get_const_nets, get_all_dag_edges, add_load_ops, get_all_dag_edges_between, \ add_store_ops, get_dag_stats __all__ = [ - "add_read_command", + "add_read_value_remote", "Net", "Op", "Node", diff --git a/tests/test_branching_stencils.py b/tests/test_branching_stencils.py index 6ab929b..f8dab41 100644 --- a/tests/test_branching_stencils.py +++ b/tests/test_branching_stencils.py @@ -1,9 +1,8 @@ from copapy import value -from copapy.backend import Store, compile_to_dag, add_read_command +from copapy.backend import Store, compile_to_dag, add_read_value_remote import copapy as cp import subprocess from copapy import _binwrite -import copapy.backend import pytest @@ -24,7 +23,7 @@ def test_compile(): out = [Store(r) for r in ret_test] - il, variables = compile_to_dag(out, copapy.generic_sdb) + il, variables = compile_to_dag(out, cp.generic_sdb) # run program command il.write_com(_binwrite.Command.RUN_PROG) @@ -32,7 +31,7 @@ def test_compile(): for v in ret_test: assert isinstance(v, value) - add_read_command(il, variables, v.net) + add_read_value_remote(il, variables, v.net) il.write_com(_binwrite.Command.END_COM) diff --git a/tests/test_compile.py b/tests/test_compile.py index 7c0c674..7fe502b 100644 --- a/tests/test_compile.py +++ b/tests/test_compile.py @@ -1,10 +1,9 @@ from copapy import NumLike -from copapy.backend import Store, compile_to_dag, add_read_command +from copapy.backend import Store, compile_to_dag, add_read_value_remote import copapy as cp import subprocess import struct from copapy import _binwrite -import copapy.backend import pytest @@ -60,14 +59,14 @@ def test_compile(): out = [Store(r) for r in ret] - il, variables = compile_to_dag(out, copapy.generic_sdb) + il, variables = compile_to_dag(out, cp.generic_sdb) # run program command il.write_com(_binwrite.Command.RUN_PROG) for v in ret: assert isinstance(v, cp.value) - add_read_command(il, variables, v.net) + add_read_value_remote(il, variables, v.net) il.write_com(_binwrite.Command.END_COM) diff --git a/tests/test_compile_aarch64.py b/tests/test_compile_aarch64.py index 2a56385..dbe25c9 100644 --- a/tests/test_compile_aarch64.py +++ b/tests/test_compile_aarch64.py @@ -1,5 +1,5 @@ from copapy import NumLike -from copapy.backend import Store, compile_to_dag, add_read_command +from copapy.backend import Store, compile_to_dag, add_read_value_remote import subprocess from copapy import _binwrite import copapy.backend as backend @@ -62,7 +62,7 @@ def test_compile(): for v in ret: assert isinstance(v, cp.value) - add_read_command(il, variables, v.net) + add_read_value_remote(il, variables, v.net) il.write_com(_binwrite.Command.END_COM) diff --git a/tests/test_compile_armv7.py b/tests/test_compile_armv7.py index fec2eab..f36a76c 100644 --- a/tests/test_compile_armv7.py +++ b/tests/test_compile_armv7.py @@ -1,5 +1,5 @@ from copapy import NumLike -from copapy.backend import Store, compile_to_dag, add_read_command +from copapy.backend import Store, compile_to_dag, add_read_value_remote import subprocess from copapy import _binwrite import copapy.backend as backend @@ -63,7 +63,7 @@ def test_compile(): for v in ret: assert isinstance(v, cp.value) - add_read_command(il, variables, v.net) + add_read_value_remote(il, variables, v.net) il.write_com(_binwrite.Command.END_COM) diff --git a/tests/test_compile_div.py b/tests/test_compile_div.py index 1133d1e..269d9ae 100644 --- a/tests/test_compile_div.py +++ b/tests/test_compile_div.py @@ -1,6 +1,6 @@ from copapy import value, NumLike -from copapy.backend import Store, compile_to_dag, add_read_command -import copapy +from copapy.backend import Store, compile_to_dag, add_read_value_remote +import copapy as cp import subprocess from copapy import _binwrite import pytest @@ -28,14 +28,14 @@ def test_compile(): out = [Store(r) for r in ret] - il, vars = compile_to_dag(out, copapy.generic_sdb) + il, vars = compile_to_dag(out, cp.generic_sdb) # run program command il.write_com(_binwrite.Command.RUN_PROG) for v in ret: assert isinstance(v, value) - add_read_command(il, vars, v.net) + add_read_value_remote(il, vars, v.net) il.write_com(_binwrite.Command.END_COM) diff --git a/tests/test_compile_math.py b/tests/test_compile_math.py index 3376068..eecb61c 100644 --- a/tests/test_compile_math.py +++ b/tests/test_compile_math.py @@ -1,9 +1,8 @@ from copapy import value -from copapy.backend import Store, compile_to_dag, add_read_command +from copapy.backend import Store, compile_to_dag, add_read_value_remote import copapy as cp import subprocess from copapy import _binwrite -import copapy.backend import pytest @@ -23,14 +22,14 @@ def test_compile_sqrt(): out = [Store(r) for r in ret] - il, variables = compile_to_dag(out, copapy.generic_sdb) + il, variables = compile_to_dag(out, cp.generic_sdb) # run program command il.write_com(_binwrite.Command.RUN_PROG) for v in ret: assert isinstance(v, value) - add_read_command(il, variables, v.net) + add_read_value_remote(il, variables, v.net) il.write_com(_binwrite.Command.END_COM) @@ -57,14 +56,14 @@ def test_compile_log(): out = [Store(r) for r in ret] - il, variables = compile_to_dag(out, copapy.generic_sdb) + il, variables = compile_to_dag(out, cp.generic_sdb) # run program command il.write_com(_binwrite.Command.RUN_PROG) for v in ret: assert isinstance(v, value) - add_read_command(il, variables, v.net) + add_read_value_remote(il, variables, v.net) il.write_com(_binwrite.Command.END_COM) @@ -91,14 +90,14 @@ def test_compile_sin(): out = [Store(r) for r in ret] - il, variables = compile_to_dag(out, copapy.generic_sdb) + il, variables = compile_to_dag(out, cp.generic_sdb) # run program command il.write_com(_binwrite.Command.RUN_PROG) for v in ret: - assert isinstance(v, copapy.value) - add_read_command(il, variables, v.net) + assert isinstance(v, cp.value) + add_read_value_remote(il, variables, v.net) il.write_com(_binwrite.Command.END_COM) diff --git a/tests/test_ops_aarch64.py b/tests/test_ops_aarch64.py index d733425..c052dcb 100644 --- a/tests/test_ops_aarch64.py +++ b/tests/test_ops_aarch64.py @@ -1,5 +1,5 @@ from copapy import NumLike, iif, value -from copapy.backend import Store, compile_to_dag, add_read_command +from copapy.backend import Store, compile_to_dag, add_read_value_remote import subprocess from copapy import _binwrite import copapy.backend as backend @@ -109,7 +109,7 @@ def test_compile(): for v in ret_test: assert isinstance(v, value) - add_read_command(dw, variables, v.net) + add_read_value_remote(dw, variables, v.net) #dw.write_com(_binwrite.Command.READ_DATA) #dw.write_int(0) diff --git a/tests/test_ops_armv6.py b/tests/test_ops_armv6.py index 0d716cb..75252fb 100644 --- a/tests/test_ops_armv6.py +++ b/tests/test_ops_armv6.py @@ -1,5 +1,5 @@ from copapy import NumLike, iif, value -from copapy.backend import Store, compile_to_dag, add_read_command +from copapy.backend import Store, compile_to_dag, add_read_value_remote import subprocess from copapy import _binwrite import copapy.backend as backend @@ -111,7 +111,7 @@ def test_compile(): for v in ret_test: assert isinstance(v, value) - add_read_command(dw, variables, v.net) + add_read_value_remote(dw, variables, v.net) #dw.write_com(_binwrite.Command.READ_DATA) #dw.write_int(0) diff --git a/tests/test_ops_armv7.py b/tests/test_ops_armv7.py index c354eff..ee0c5b7 100644 --- a/tests/test_ops_armv7.py +++ b/tests/test_ops_armv7.py @@ -1,5 +1,5 @@ from copapy import NumLike, iif, value -from copapy.backend import Store, compile_to_dag, add_read_command +from copapy.backend import Store, compile_to_dag, add_read_value_remote import subprocess from copapy import _binwrite import copapy.backend as backend @@ -111,7 +111,7 @@ def test_compile(): for v in ret_test: assert isinstance(v, value) - add_read_command(dw, variables, v.net) + add_read_value_remote(dw, variables, v.net) #dw.write_com(_binwrite.Command.READ_DATA) #dw.write_int(0) diff --git a/tests/test_ops_x86.py b/tests/test_ops_x86.py index c6d0643..c6f784b 100644 --- a/tests/test_ops_x86.py +++ b/tests/test_ops_x86.py @@ -1,5 +1,5 @@ from copapy import NumLike, iif, value -from copapy.backend import Store, compile_to_dag, add_read_command +from copapy.backend import Store, compile_to_dag, add_read_value_remote import subprocess from copapy import _binwrite import copapy.backend as backend @@ -122,7 +122,7 @@ def test_compile(): for v in ret_test: assert isinstance(v, value) - add_read_command(dw, variables, v.net) + add_read_value_remote(dw, variables, v.net) #dw.write_com(_binwrite.Command.READ_DATA) #dw.write_int(0) @@ -196,7 +196,7 @@ def test_vector_compile(): for v in ret: assert isinstance(v, cp.value) - add_read_command(il, variables, v.net) + add_read_value_remote(il, variables, v.net) il.write_com(_binwrite.Command.END_COM) @@ -258,7 +258,7 @@ def test_sinus(): for v in ret_test: assert isinstance(v, value) - add_read_command(dw, variables, v.net) + add_read_value_remote(dw, variables, v.net) #dw.write_com(_binwrite.Command.READ_DATA) #dw.write_int(0) From 3ec0ba10c32aea147442796ffc4ee73936f682ba Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sun, 25 Jan 2026 17:13:01 +0100 Subject: [PATCH 02/27] Skiping qemo based ARM32 bit tests on windows since they are not compatible to wsl1 --- tests/test_compile_armv7.py | 71 ++++++++------ tests/test_ops_armv6.py | 180 ++++++++++++++++++++++++------------ tests/test_ops_armv7.py | 178 +++++++++++++++++++++++------------ 3 files changed, 284 insertions(+), 145 deletions(-) diff --git a/tests/test_compile_armv7.py b/tests/test_compile_armv7.py index f36a76c..4d2ee4d 100644 --- a/tests/test_compile_armv7.py +++ b/tests/test_compile_armv7.py @@ -1,32 +1,45 @@ -from copapy import NumLike -from copapy.backend import Store, compile_to_dag, add_read_value_remote -import subprocess -from copapy import _binwrite -import copapy.backend as backend -import copapy as cp import os +import subprocess import warnings + import pytest -if os.name == 'nt': +import copapy as cp +import copapy.backend as backend +from copapy import NumLike, _binwrite +from copapy.backend import Store, add_read_value_remote, compile_to_dag + +if os.name == "nt": # On Windows wsl and qemu-user is required: # sudo apt install qemu-user - qemu_command = ['wsl', 'qemu-arm'] + qemu_command = ["wsl", "qemu-arm"] else: - qemu_command = ['qemu-arm'] + qemu_command = ["qemu-arm"] def run_command(command: list[str]) -> str: - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8', check=False) - assert result.returncode != 11, f"SIGSEGV (segmentation fault)\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" - assert result.returncode == 0, f"\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf8", + check=False, + ) + assert result.returncode != 11, ( + f"SIGSEGV (segmentation fault)\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + ) + assert result.returncode == 0, ( + f"\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + ) return result.stdout def check_for_qemu() -> bool: - command = qemu_command + ['--version'] + command = qemu_command + ["--version"] try: - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + result = subprocess.run( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False + ) except Exception: return False return result.returncode == 0 @@ -54,12 +67,12 @@ def test_compile(): out = [Store(r) for r in ret] - sdb = backend.stencil_db_from_package('armv7') + sdb = backend.stencil_db_from_package("armv7") il, variables = compile_to_dag(out, sdb) # run program command il.write_com(_binwrite.Command.RUN_PROG) - #il.write_com(_binwrite.Command.DUMP_CODE) + # il.write_com(_binwrite.Command.DUMP_CODE) for v in ret: assert isinstance(v, cp.value) @@ -67,23 +80,29 @@ def test_compile(): il.write_com(_binwrite.Command.END_COM) - #print('* Data to runner:') - #il.print() + # print('* Data to runner:') + # il.print() - il.to_file('build/runner/test-armv7.copapy') + il.to_file("build/runner/test-armv7.copapy") if not check_for_qemu(): - warnings.warn("qemu-armv7 not found, aarch64 test skipped!", UserWarning) - elif not os.path.isfile('build/runner/coparun-armv7'): + warnings.warn("qemu-armv7 not found, test skipped!", UserWarning) + elif "wsl" in qemu_command: + warnings.warn("qemu-armv7 seams not work on wsl1, test skipped!", UserWarning) + elif not os.path.isfile("build/runner/coparun-armv7"): warnings.warn("armv7 runner not found, aarch64 test skipped!", UserWarning) else: - command = ['build/runner/coparun-armv7', 'build/runner/test-armv7.copapy', 'build/runner/test-armv7.copapy.bin'] + command = [ + "build/runner/coparun-armv7", + "build/runner/test-armv7.copapy", + "build/runner/test-armv7.copapy.bin", + ] result = run_command(qemu_command + command) - print('* Output from runner:\n--') + print("* Output from runner:\n--") print(result) - print('--') + print("--") - assert 'Return value: 1' in result + assert "Return value: 1" in result # Compare to x86_64 reference results assert " size=4 data=24 00 00 00" in result @@ -92,5 +111,5 @@ def test_compile(): if __name__ == "__main__": - #test_example() + # test_example() test_compile() diff --git a/tests/test_ops_armv6.py b/tests/test_ops_armv6.py index 75252fb..4be1cfe 100644 --- a/tests/test_ops_armv6.py +++ b/tests/test_ops_armv6.py @@ -1,21 +1,22 @@ -from copapy import NumLike, iif, value -from copapy.backend import Store, compile_to_dag, add_read_value_remote -import subprocess -from copapy import _binwrite -import copapy.backend as backend import os -import warnings import re import struct -import pytest -import copapy as cp +import subprocess +import warnings -if os.name == 'nt': +import pytest + +import copapy as cp +import copapy.backend as backend +from copapy import NumLike, _binwrite, iif, value +from copapy.backend import Store, add_read_value_remote, compile_to_dag + +if os.name == "nt": # On Windows wsl and qemu-user is required: # sudo apt install qemu-user - qemu_command = ['wsl', 'qemu-arm'] + qemu_command = ["wsl", "qemu-arm"] else: - qemu_command = ['qemu-arm'] + qemu_command = ["qemu-arm"] def parse_results(log_text: str) -> dict[int, bytes]: @@ -24,8 +25,8 @@ def parse_results(log_text: str) -> dict[int, bytes]: var_dict: dict[int, bytes] = {} for match in matches: - value_str: list[str] = match.group(3).strip().split(' ') - #print('--', value_str) + value_str: list[str] = match.group(3).strip().split(" ") + # print('--', value_str) value = bytes(int(v, base=16) for v in value_str) if len(value) <= 8: var_dict[int(match.group(1))] = value @@ -34,26 +35,49 @@ def parse_results(log_text: str) -> dict[int, bytes]: def run_command(command: list[str]) -> str: - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8', check=False) - assert result.returncode != 11, f"SIGSEGV (segmentation fault)\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" - assert result.returncode == 0, f"\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf8", + check=False, + ) + assert result.returncode != 11, ( + f"SIGSEGV (segmentation fault)\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + ) + assert result.returncode == 0, ( + f"\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + ) return result.stdout def check_for_qemu() -> bool: - command = qemu_command + ['--version'] + command = qemu_command + ["--version"] try: - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + result = subprocess.run( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False + ) except Exception: return False return result.returncode == 0 def function1(c1: NumLike) -> list[NumLike]: - return [c1 / 4, c1 / -4, c1 // 4, c1 // -4, (c1 * -1) // 4, - c1 * 4, c1 * -4, - c1 + 4, c1 - 4, - c1 > 2, c1 > 100, c1 < 4, c1 < 100] + return [ + c1 / 4, + c1 / -4, + c1 // 4, + c1 // -4, + (c1 * -1) // 4, + c1 * 4, + c1 * -4, + c1 + 4, + c1 - 4, + c1 > 2, + c1 > 100, + c1 < 4, + c1 < 100, + ] def function2(c1: NumLike) -> list[NumLike]: @@ -77,11 +101,13 @@ def function6(c1: NumLike) -> list[NumLike]: def iiftests(c1: NumLike) -> list[NumLike]: - return [iif(c1 > 5, 8, 9), - iif(c1 < 5, 8.5, 9.5), - iif(1 > 5, 3.3, 8.8) + c1, - iif(1 < 5, c1 * 3.3, 8.8), - iif(c1 < 5, c1 * 3.3, 8.8)] + return [ + iif(c1 > 5, 8, 9), + iif(c1 < 5, 8.5, 9.5), + iif(1 > 5, 3.3, 8.8) + c1, + iif(1 < 5, c1 * 3.3, 8.8), + iif(c1 < 5, c1 * 3.3, 8.8), + ] @pytest.mark.runner @@ -90,59 +116,90 @@ def test_compile(): c_f = value(1.111) c_b = value(True) - ret_test = function1(c_i) + function1(c_f) + function2(c_i) + function2(c_f) + function3(c_i) + function4(c_i) + function5(c_b) + [value(9) % 2] + iiftests(c_i) + iiftests(c_f) + [cp.asin(c_i/10)] - ret_ref = function1(9) + function1(1.111) + function2(9) + function2(1.111) + function3(9) + function4(9) + function5(True) + [9 % 2] + iiftests(9) + iiftests(1.111) + [cp.asin(9/10)] + ret_test = ( + function1(c_i) + + function1(c_f) + + function2(c_i) + + function2(c_f) + + function3(c_i) + + function4(c_i) + + function5(c_b) + + [value(9) % 2] + + iiftests(c_i) + + iiftests(c_f) + + [cp.asin(c_i / 10)] + ) + ret_ref = ( + function1(9) + + function1(1.111) + + function2(9) + + function2(1.111) + + function3(9) + + function4(9) + + function5(True) + + [9 % 2] + + iiftests(9) + + iiftests(1.111) + + [cp.asin(9 / 10)] + ) - #ret_test = (c_i * 100 // 5, c_f * 10 // 5) - #ret_ref = (9 * 100 // 5, 1.111 * 10 // 5) + # ret_test = (c_i * 100 // 5, c_f * 10 // 5) + # ret_ref = (9 * 100 // 5, 1.111 * 10 // 5) out = [Store(r) for r in ret_test] - sdb = backend.stencil_db_from_package('armv6') + sdb = backend.stencil_db_from_package("armv6") dw, variables = compile_to_dag(out, sdb) - #dw.write_com(_binwrite.Command.READ_DATA) - #dw.write_int(0) - #dw.write_int(28) + # dw.write_com(_binwrite.Command.READ_DATA) + # dw.write_int(0) + # dw.write_int(28) # run program command dw.write_com(_binwrite.Command.RUN_PROG) - #dw.write_com(_binwrite.Command.DUMP_CODE) + # dw.write_com(_binwrite.Command.DUMP_CODE) for v in ret_test: assert isinstance(v, value) add_read_value_remote(dw, variables, v.net) - #dw.write_com(_binwrite.Command.READ_DATA) - #dw.write_int(0) - #dw.write_int(28) + # dw.write_com(_binwrite.Command.READ_DATA) + # dw.write_int(0) + # dw.write_int(28) dw.write_com(_binwrite.Command.END_COM) - #print('* Data to runner:') - #dw.print() + # print('* Data to runner:') + # dw.print() - dw.to_file('build/runner/test-armv6.copapy') + dw.to_file("build/runner/test-armv6.copapy") if not check_for_qemu(): warnings.warn("qemu-armv6 not found, armv6 test skipped!", UserWarning) return - if not os.path.isfile('build/runner/coparun-armv6'): + if "wsl" in qemu_command: + warnings.warn("qemu-armv6 seams not work on wsl1, test skipped!", UserWarning) + return + if not os.path.isfile("build/runner/coparun-armv6"): warnings.warn("armv6 runner not found, armv6 test skipped!", UserWarning) return - command = qemu_command + ['build/runner/coparun-armv6', 'build/runner/test-armv6.copapy'] + ['build/runner/test-armv6.copapy.bin'] - #try: + command = ( + qemu_command + + ["build/runner/coparun-armv6", "build/runner/test-armv6.copapy"] + + ["build/runner/test-armv6.copapy.bin"] + ) + # try: result = run_command(command) - #except FileNotFoundError: + # except FileNotFoundError: # warnings.warn(f"Test skipped, executable not found.", UserWarning) # return - #print('* Output from runner:\n--') - #print(result) - #print('--') + # print('* Output from runner:\n--') + # print(result) + # print('--') - assert 'Return value: 1' in result + assert "Return value: 1" in result result_data = parse_results(result) @@ -150,22 +207,25 @@ def test_compile(): assert isinstance(test, value) address = variables[test.net][0] data = result_data[address] - if test.dtype == 'int': + if test.dtype == "int": val = int.from_bytes(data, sdb.byteorder, signed=True) - elif test.dtype == 'bool': + elif test.dtype == "bool": val = bool.from_bytes(data, sdb.byteorder) - elif test.dtype == 'float': - en = {'little': '<', 'big': '>'}[sdb.byteorder] - val = struct.unpack(en + 'f', data)[0] + elif test.dtype == "float": + en = {"little": "<", "big": ">"}[sdb.byteorder] + val = struct.unpack(en + "f", data)[0] assert isinstance(val, float) else: raise Exception(f"Unknown type: {test.dtype}") - print('+', val, ref, test.dtype, f" addr={address}") + print("+", val, ref, test.dtype, f" addr={address}") for t in (int, float, bool): - assert isinstance(val, t) == isinstance(ref, t), f"Result type does not match for {val} and {ref}" - assert val == pytest.approx(ref, 1e-5), f"Result does not match: {val} and reference: {ref}" # pyright: ignore[reportUnknownMemberType] + assert isinstance(val, t) == isinstance(ref, t), ( + f"Result type does not match for {val} and {ref}" + ) + assert val == pytest.approx(ref, 1e-5), ( + f"Result does not match: {val} and reference: {ref}" + ) # pyright: ignore[reportUnknownMemberType] if __name__ == "__main__": - #test_compile() - test_slow_31bit_int_list_hash() + test_compile() diff --git a/tests/test_ops_armv7.py b/tests/test_ops_armv7.py index ee0c5b7..ea2b4b0 100644 --- a/tests/test_ops_armv7.py +++ b/tests/test_ops_armv7.py @@ -1,21 +1,22 @@ -from copapy import NumLike, iif, value -from copapy.backend import Store, compile_to_dag, add_read_value_remote -import subprocess -from copapy import _binwrite -import copapy.backend as backend import os -import warnings import re import struct -import pytest -import copapy as cp +import subprocess +import warnings -if os.name == 'nt': +import pytest + +import copapy as cp +import copapy.backend as backend +from copapy import NumLike, _binwrite, iif, value +from copapy.backend import Store, add_read_value_remote, compile_to_dag + +if os.name == "nt": # On Windows wsl and qemu-user is required: # sudo apt install qemu-user - qemu_command = ['wsl', 'qemu-arm'] + qemu_command = ["wsl", "qemu-arm"] else: - qemu_command = ['qemu-arm'] + qemu_command = ["qemu-arm"] def parse_results(log_text: str) -> dict[int, bytes]: @@ -24,8 +25,8 @@ def parse_results(log_text: str) -> dict[int, bytes]: var_dict: dict[int, bytes] = {} for match in matches: - value_str: list[str] = match.group(3).strip().split(' ') - #print('--', value_str) + value_str: list[str] = match.group(3).strip().split(" ") + # print('--', value_str) value = bytes(int(v, base=16) for v in value_str) if len(value) <= 8: var_dict[int(match.group(1))] = value @@ -34,26 +35,49 @@ def parse_results(log_text: str) -> dict[int, bytes]: def run_command(command: list[str]) -> str: - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8', check=False) - assert result.returncode != 11, f"SIGSEGV (segmentation fault)\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" - assert result.returncode == 0, f"\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf8", + check=False, + ) + assert result.returncode != 11, ( + f"SIGSEGV (segmentation fault)\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + ) + assert result.returncode == 0, ( + f"\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + ) return result.stdout def check_for_qemu() -> bool: - command = qemu_command + ['--version'] + command = qemu_command + ["--version"] try: - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + result = subprocess.run( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False + ) except Exception: return False return result.returncode == 0 def function1(c1: NumLike) -> list[NumLike]: - return [c1 / 4, c1 / -4, c1 // 4, c1 // -4, (c1 * -1) // 4, - c1 * 4, c1 * -4, - c1 + 4, c1 - 4, - c1 > 2, c1 > 100, c1 < 4, c1 < 100] + return [ + c1 / 4, + c1 / -4, + c1 // 4, + c1 // -4, + (c1 * -1) // 4, + c1 * 4, + c1 * -4, + c1 + 4, + c1 - 4, + c1 > 2, + c1 > 100, + c1 < 4, + c1 < 100, + ] def function2(c1: NumLike) -> list[NumLike]: @@ -77,11 +101,13 @@ def function6(c1: NumLike) -> list[NumLike]: def iiftests(c1: NumLike) -> list[NumLike]: - return [iif(c1 > 5, 8, 9), - iif(c1 < 5, 8.5, 9.5), - iif(1 > 5, 3.3, 8.8) + c1, - iif(1 < 5, c1 * 3.3, 8.8), - iif(c1 < 5, c1 * 3.3, 8.8)] + return [ + iif(c1 > 5, 8, 9), + iif(c1 < 5, 8.5, 9.5), + iif(1 > 5, 3.3, 8.8) + c1, + iif(1 < 5, c1 * 3.3, 8.8), + iif(c1 < 5, c1 * 3.3, 8.8), + ] @pytest.mark.runner @@ -90,59 +116,90 @@ def test_compile(): c_f = value(1.111) c_b = value(True) - ret_test = function1(c_i) + function1(c_f) + function2(c_i) + function2(c_f) + function3(c_i) + function4(c_i) + function5(c_b) + [value(9) % 2] + iiftests(c_i) + iiftests(c_f) + [cp.asin(c_i/10)] - ret_ref = function1(9) + function1(1.111) + function2(9) + function2(1.111) + function3(9) + function4(9) + function5(True) + [9 % 2] + iiftests(9) + iiftests(1.111) + [cp.asin(9/10)] + ret_test = ( + function1(c_i) + + function1(c_f) + + function2(c_i) + + function2(c_f) + + function3(c_i) + + function4(c_i) + + function5(c_b) + + [value(9) % 2] + + iiftests(c_i) + + iiftests(c_f) + + [cp.asin(c_i / 10)] + ) + ret_ref = ( + function1(9) + + function1(1.111) + + function2(9) + + function2(1.111) + + function3(9) + + function4(9) + + function5(True) + + [9 % 2] + + iiftests(9) + + iiftests(1.111) + + [cp.asin(9 / 10)] + ) - #ret_test = (c_i * 100 // 5, c_f * 10 // 5) - #ret_ref = (9 * 100 // 5, 1.111 * 10 // 5) + # ret_test = (c_i * 100 // 5, c_f * 10 // 5) + # ret_ref = (9 * 100 // 5, 1.111 * 10 // 5) out = [Store(r) for r in ret_test] - sdb = backend.stencil_db_from_package('armv7') + sdb = backend.stencil_db_from_package("armv7") dw, variables = compile_to_dag(out, sdb) - #dw.write_com(_binwrite.Command.READ_DATA) - #dw.write_int(0) - #dw.write_int(28) + # dw.write_com(_binwrite.Command.READ_DATA) + # dw.write_int(0) + # dw.write_int(28) # run program command dw.write_com(_binwrite.Command.RUN_PROG) - #dw.write_com(_binwrite.Command.DUMP_CODE) + # dw.write_com(_binwrite.Command.DUMP_CODE) for v in ret_test: assert isinstance(v, value) add_read_value_remote(dw, variables, v.net) - #dw.write_com(_binwrite.Command.READ_DATA) - #dw.write_int(0) - #dw.write_int(28) + # dw.write_com(_binwrite.Command.READ_DATA) + # dw.write_int(0) + # dw.write_int(28) dw.write_com(_binwrite.Command.END_COM) - #print('* Data to runner:') - #dw.print() + # print('* Data to runner:') + # dw.print() - dw.to_file('build/runner/test-armv7.copapy') + dw.to_file("build/runner/test-armv7.copapy") if not check_for_qemu(): warnings.warn("qemu-armv7 not found, armv7 test skipped!", UserWarning) return - if not os.path.isfile('build/runner/coparun-armv7'): + if "wsl" in qemu_command: + warnings.warn("qemu-armv7 seams not work on wsl1, test skipped!", UserWarning) + return + if not os.path.isfile("build/runner/coparun-armv7"): warnings.warn("armv7 runner not found, armv7 test skipped!", UserWarning) return - command = qemu_command + ['build/runner/coparun-armv7', 'build/runner/test-armv7.copapy'] + ['build/runner/test-armv7.copapy.bin'] - #try: + command = ( + qemu_command + + ["build/runner/coparun-armv7", "build/runner/test-armv7.copapy"] + + ["build/runner/test-armv7.copapy.bin"] + ) + # try: result = run_command(command) - #except FileNotFoundError: + # except FileNotFoundError: # warnings.warn(f"Test skipped, executable not found.", UserWarning) # return - print('* Output from runner:\n--') + print("* Output from runner:\n--") print(result) - print('--') + print("--") - assert 'Return value: 1' in result + assert "Return value: 1" in result result_data = parse_results(result) @@ -150,22 +207,25 @@ def test_compile(): assert isinstance(test, value) address = variables[test.net][0] data = result_data[address] - if test.dtype == 'int': + if test.dtype == "int": val = int.from_bytes(data, sdb.byteorder, signed=True) - elif test.dtype == 'bool': + elif test.dtype == "bool": val = bool.from_bytes(data, sdb.byteorder) - elif test.dtype == 'float': - en = {'little': '<', 'big': '>'}[sdb.byteorder] - val = struct.unpack(en + 'f', data)[0] + elif test.dtype == "float": + en = {"little": "<", "big": ">"}[sdb.byteorder] + val = struct.unpack(en + "f", data)[0] assert isinstance(val, float) else: raise Exception(f"Unknown type: {test.dtype}") - print('+', val, ref, test.dtype, f" addr={address}") + print("+", val, ref, test.dtype, f" addr={address}") for t in (int, float, bool): - assert isinstance(val, t) == isinstance(ref, t), f"Result type does not match for {val} and {ref}" - assert val == pytest.approx(ref, 1e-5), f"Result does not match: {val} and reference: {ref}" # pyright: ignore[reportUnknownMemberType] + assert isinstance(val, t) == isinstance(ref, t), ( + f"Result type does not match for {val} and {ref}" + ) + assert val == pytest.approx(ref, 1e-5), ( + f"Result does not match: {val} and reference: {ref}" + ) # pyright: ignore[reportUnknownMemberType] if __name__ == "__main__": - #test_example() - test_slow_31bit_int_list_hash() + test_compile() From 7131483a22d8a057f6bc73d95b3a0a4b43025e07 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sun, 25 Jan 2026 17:16:33 +0100 Subject: [PATCH 03/27] CI: Added -static for building the runner --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6212351..349f366 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,7 @@ jobs: - name: Compile coparun run: | mkdir -p build/runner - gcc -O3 -DENABLE_BASIC_LOGGING -o build/runner/coparun src/coparun/runmem.c src/coparun/coparun.c src/coparun/mem_man.c + gcc -O3 -static -DENABLE_BASIC_LOGGING -o build/runner/coparun src/coparun/runmem.c src/coparun/coparun.c src/coparun/mem_man.c - name: Generate debug asm files if: strategy.job-index == 0 @@ -153,7 +153,7 @@ jobs: docker run --rm -v $PWD:/app -w /app --platform linux/arm64 ghcr.io/nonannet/arm64_test:1 \ bash -lc "pip install . && \ mkdir -p build/runner && \ - gcc -O3 -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \ + gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \ src/coparun/coparun.c src/coparun/mem_man.c && \ pytest && \ bash tools/create_asm.sh" @@ -182,7 +182,7 @@ jobs: docker run --rm -v $PWD:/app -w /app --platform linux/arm/v6 ghcr.io/nonannet/armv6_test:1 \ bash -lc "pip install . && \ mkdir -p build/runner && \ - gcc -O3 -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \ + gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \ src/coparun/coparun.c src/coparun/mem_man.c && \ pytest && \ bash tools/create_asm.sh" @@ -211,7 +211,7 @@ jobs: docker run --rm -v $PWD:/app -w /app --platform linux/arm/v7 ghcr.io/nonannet/armv7_test:1 \ bash -lc "pip install . && \ mkdir -p build/runner && \ - gcc -O3 -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \ + gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \ src/coparun/coparun.c src/coparun/mem_man.c && \ pytest && \ bash tools/create_asm.sh" From 58120f292c7b4ee3055b90a84b1083176b8c27ac Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 26 Jan 2026 23:57:15 +0100 Subject: [PATCH 04/27] partial arm thumb implementation added --- .gitignore | 1 + src/copapy/_binwrite.py | 4 +- src/copapy/_compiler.py | 9 +- src/copapy/_stencils.py | 26 +++++- src/coparun/runmem.c | 45 +++++++++ src/coparun/runmem.h | 1 + tests/test_ops_armv7thunb.py | 171 +++++++++++++++++++++++++++++++++++ tools/build.sh | 37 +++++++- tools/inspect.sh | 2 + 9 files changed, 287 insertions(+), 9 deletions(-) create mode 100644 tests/test_ops_armv7thunb.py diff --git a/.gitignore b/.gitignore index aaa1e38..dfe5d47 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ core *.log docs/source/start.md /src/copapy/_version.py +sketch*.py diff --git a/src/copapy/_binwrite.py b/src/copapy/_binwrite.py index ff77918..a12d82b 100644 --- a/src/copapy/_binwrite.py +++ b/src/copapy/_binwrite.py @@ -6,7 +6,9 @@ ByteOrder = Literal['little', 'big'] Command = Enum('Command', [('ALLOCATE_DATA', 1), ('COPY_DATA', 2), ('ALLOCATE_CODE', 3), ('COPY_CODE', 4), - ('PATCH_FUNC', 0x1000), ('PATCH_OBJECT', 0x2000), + ('PATCH_FUNC', 0x1000), + ('PATCH_FUNC_ARM32_THM', 0x1005), + ('PATCH_OBJECT', 0x2000), ('PATCH_OBJECT_HI21', 0x2001), ('PATCH_OBJECT_ABS', 0x2002), ('PATCH_OBJECT_REL', 0x2003), diff --git a/src/copapy/_compiler.py b/src/copapy/_compiler.py index cbade28..cd42d41 100644 --- a/src/copapy/_compiler.py +++ b/src/copapy/_compiler.py @@ -393,13 +393,14 @@ def compile_to_dag(node_list: Iterable[Node], sdb: stencil_database) -> tuple[bi # assemble stencils to main program and patch stencils data = sdb.get_function_code('entry_function_shell', 'start') data_list.append(data) + print(f"* entry_function_shell (0) " + ' '.join(f'{d:02X}' for d in data)) offset = aux_func_len + len(data) for associated_net, node in extended_output_ops: assert node.name in sdb.stencil_definitions, f"- Warning: {node.name} stencil not found" data = sdb.get_stencil_code(node.name) data_list.append(data) - #print(f"* {node.name} ({offset}) " + ' '.join(f'{d:02X}' for d in data)) + print(f"* {node.name} ({offset}) " + ' '.join(f'{d:02X}' for d in data)) for reloc in sdb.get_relocations(node.name, stencil=True): if reloc.target_symbol_info in ('STT_OBJECT', 'STT_NOTYPE', 'STT_SECTION'): @@ -451,10 +452,8 @@ def compile_to_dag(node_list: Iterable[Node], sdb: stencil_database) -> tuple[bi #print('--> ', name, list(sdb.get_relocations(name))) for reloc in sdb.get_relocations(name): - #assert reloc.target_symbol_info != 'STT_FUNC', "Not tested yet!" - if not reloc.target_section_index: - assert reloc.pelfy_reloc.type == 'R_ARM_V4BX' + assert reloc.pelfy_reloc.type == 'R_ARM_V4BX', (reloc.pelfy_reloc.type, name) elif reloc.target_symbol_info in {'STT_OBJECT', 'STT_NOTYPE', 'STT_SECTION'}: # Patch constants/variable addresses on heap @@ -489,6 +488,6 @@ def compile_to_dag(node_list: Iterable[Node], sdb: stencil_database) -> tuple[bi dw.write_int(patch.value, signed=True) dw.write_com(binw.Command.ENTRY_POINT) - dw.write_int(aux_func_len) + dw.write_int(aux_func_len + sdb.thumb_mode) return dw, variables diff --git a/src/copapy/_stencils.py b/src/copapy/_stencils.py index b7864c0..f464b3e 100644 --- a/src/copapy/_stencils.py +++ b/src/copapy/_stencils.py @@ -121,6 +121,7 @@ class stencil_database(): var_size (dict[str, int]): dictionary of object names and their sizes byteorder (ByteOrder): byte order of the ELF file elf (elf_file): the loaded ELF file + thumb_mode (int): 1 if ARM in thumb mode, 0 otherwise """ def __init__(self, obj_file: str | bytes): @@ -147,6 +148,11 @@ class stencil_database(): # if s.info == 'STT_OBJECT'} self.byteorder: ByteOrder = self.elf.byteorder + self.arm = '.ARM.attributes' in self.elf.sections + + # Returns 1 for ARM in thumb mode, 0 otherwise + self.thumb_mode = self.elf.symbols['entry_function_shell'].fields['st_value'] & 1 + #for name in self.function_definitions.keys(): # sym = self.elf.symbols[name] # sym.relocations @@ -314,6 +320,13 @@ class stencil_database(): patch_value = symbol_address + pr.fields['r_addend'] symbol_type = symbol_type + 0x03 # Relative to data section + elif pr.type.endswith('_THM_JUMP24') or pr.type.endswith('_THM_CALL'): + # R_ARM_THM_JUMP24 + # S + A - P + #assert pr.fields['r_addend'] == 0, pr.fields['r_addend'] + patch_value = symbol_address - (patch_offset + 4) #+ pr.fields['r_addend'] + symbol_type = symbol_type + 0x05 # PATCH_FUNC_ARM32_THM + else: raise NotImplementedError(f"Relocation type {pr.type} in {relocation.pelfy_reloc.target_section.name} pointing to {relocation.pelfy_reloc.symbol.name} not implemented") @@ -334,7 +347,11 @@ class stencil_database(): func = self.elf.symbols[name] start_stencil, end_stencil = get_stencil_position(func) assert func.section - start_index = func.section['sh_offset'] + func['st_value'] + start_stencil + + # For arm functions, mask out the thumb mode bit + function_offset = func.fields['st_value'] & ~int(self.arm) + + start_index = func.section['sh_offset'] + function_offset + start_stencil lengths = end_stencil - start_stencil self._stencil_cache[name] = (start_index, lengths) @@ -403,6 +420,13 @@ class stencil_database(): func = self.elf.symbols[name] assert func.info == 'STT_FUNC', f"{name} is not a function" + # For arm functions, mask out the thumb mode bit + function_offset = func.fields['st_value'] & ~int(self.arm) + + start_index = func.section['sh_offset'] + function_offset + lengths = end_stencil - start_stencil + + #return self.elf.read_bytes(start_index, func.fields['st_size']) if part == 'start': index = get_last_call_in_function(func) return func.data[:index] diff --git a/src/coparun/runmem.c b/src/coparun/runmem.c index 1165956..e5180e5 100644 --- a/src/coparun/runmem.c +++ b/src/coparun/runmem.c @@ -50,6 +50,41 @@ void patch_arm32_abs(uint8_t *patch_addr, uint32_t imm16) *((uint32_t *)patch_addr) = instr; } +void patch_arm_thm_jump24(uint8_t *patch_addr, int32_t imm24) +{ + // Read the 32-bit instruction (two halfwords) + uint16_t *instr16 = (uint16_t *)patch_addr; + uint16_t first_half = instr16[0]; + uint16_t second_half = instr16[1]; + + // Thumb branch instructions always have LSB = 0 (halfword aligned) + // The imm24 offset in Thumb is shifted right by 1 when encoded + int32_t offset = imm24 >> 1; + + // Split into S, J1, J2, imm10, imm11 + uint32_t S = (offset >> 23) & 0x1; + uint32_t I1 = (offset >> 22) & 0x1; + uint32_t I2 = (offset >> 21) & 0x1; + uint32_t imm10 = (offset >> 11) & 0x3FF; + uint32_t imm11 = offset & 0x7FF; + + // Re-encode J1 and J2 + uint32_t J1 = (~(I1 ^ S)) & 0x1; + uint32_t J2 = (~(I2 ^ S)) & 0x1; + + // Clear old imm fields + first_half &= 0xF800; // Keep upper 5 bits + second_half &= 0xD000; // Keep upper 5 bits + + // Set new imm fields + first_half |= (S << 10) | imm10; + second_half |= (J1 << 13) | (J2 << 11) | imm11; + + // Write back + instr16[0] = first_half; + instr16[1] = second_half; +} + void free_memory(runmem_t *context) { deallocate_memory(context->executable_memory, context->executable_memory_len); deallocate_memory(context->data_memory, context->data_memory_len); @@ -180,6 +215,16 @@ int parse_commands(runmem_t *context, uint8_t *bytes) { patch_arm32_abs(context->executable_memory + offs, (uint32_t)((uintptr_t)(context->data_memory + value) & patch_mask) / (uint32_t)patch_scale); break; + case PATCH_FUNC_ARM32_THM: + offs = *(uint32_t*)bytes; bytes += 4; + patch_mask = *(uint32_t*)bytes; bytes += 4; + patch_scale = *(int32_t*)bytes; bytes += 4; + value = *(int32_t*)bytes; bytes += 4; + LOG("PATCH_FUNC_ARM32_THM patch_offs=%i patch_mask=%#08x scale=%i value=%i\n", + offs, patch_mask, patch_scale, value); + patch_arm_thm_jump24(context->executable_memory + offs, value); + break; + case ENTRY_POINT: rel_entr_point = *(uint32_t*)bytes; bytes += 4; context->entr_point = (entry_point_t)(context->executable_memory + rel_entr_point); diff --git a/src/coparun/runmem.h b/src/coparun/runmem.h index 31e9905..371f2f3 100644 --- a/src/coparun/runmem.h +++ b/src/coparun/runmem.h @@ -20,6 +20,7 @@ #define ALLOCATE_CODE 3 #define COPY_CODE 4 #define PATCH_FUNC 0x1000 +#define PATCH_FUNC_ARM32_THM 0x1005 #define PATCH_OBJECT 0x2000 #define PATCH_OBJECT_HI21 0x2001 #define PATCH_OBJECT_ABS 0x2002 diff --git a/tests/test_ops_armv7thunb.py b/tests/test_ops_armv7thunb.py new file mode 100644 index 0000000..ac67e51 --- /dev/null +++ b/tests/test_ops_armv7thunb.py @@ -0,0 +1,171 @@ +from copapy import NumLike, iif, value +from copapy.backend import Store, compile_to_dag, add_read_value_remote +import subprocess +from copapy import _binwrite +import copapy.backend as backend +import os +import warnings +import re +import struct +import pytest +import copapy as cp + +if os.name == 'nt': + # On Windows wsl and qemu-user is required: + # sudo apt install qemu-user + qemu_command = ['wsl', 'qemu-arm'] +else: + qemu_command = ['qemu-arm'] + + +def parse_results(log_text: str) -> dict[int, bytes]: + regex = r"^READ_DATA offs=(\d*) size=(\d*) data=(.*)$" + matches = re.finditer(regex, log_text, re.MULTILINE) + var_dict: dict[int, bytes] = {} + + for match in matches: + value_str: list[str] = match.group(3).strip().split(' ') + #print('--', value_str) + value = bytes(int(v, base=16) for v in value_str) + if len(value) <= 8: + var_dict[int(match.group(1))] = value + + return var_dict + + +def run_command(command: list[str]) -> str: + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8', check=False) + assert result.returncode != 11, f"SIGSEGV (segmentation fault)\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + assert result.returncode == 0, f"\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + return result.stdout + + +def check_for_qemu() -> bool: + command = qemu_command + ['--version'] + try: + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + except Exception: + return False + return result.returncode == 0 + + +def function1(c1: NumLike) -> list[NumLike]: + return [c1 / 4, c1 / -4, c1 // 4, c1 // -4, (c1 * -1) // 4, + c1 * 4, c1 * -4, + c1 + 4, c1 - 4, + c1 > 2, c1 > 100, c1 < 4, c1 < 100] + + +def function2(c1: NumLike) -> list[NumLike]: + return [c1 * 4.44, c1 * -4.44] + + +def function3(c1: NumLike) -> list[NumLike]: + return [c1 / 4] + + +def function4(c1: NumLike) -> list[NumLike]: + return [c1 == 9, c1 == 4, c1 != 9, c1 != 4] + + +def function5(c1: NumLike) -> list[NumLike]: + return [c1 == True, c1 == False, c1 != True, c1 != False, c1 / 2, c1 + 2] + + +def function6(c1: NumLike) -> list[NumLike]: + return [c1 == True] + + +def iiftests(c1: NumLike) -> list[NumLike]: + return [iif(c1 > 5, 8, 9), + iif(c1 < 5, 8.5, 9.5), + iif(1 > 5, 3.3, 8.8) + c1, + iif(1 < 5, c1 * 3.3, 8.8), + iif(c1 < 5, c1 * 3.3, 8.8)] + + +@pytest.mark.runner +def test_compile(): + c_i = value(9) + c_f = value(1.111) + c_b = value(True) + + #ret_test = function1(c_i) + function1(c_f) + function2(c_i) + function2(c_f) + function3(c_i) + function4(c_i) + function5(c_b) + [value(9) % 2] + iiftests(c_i) + iiftests(c_f) + [cp.asin(c_i/10)] + #ret_ref = function1(9) + function1(1.111) + function2(9) + function2(1.111) + function3(9) + function4(9) + function5(True) + [9 % 2] + iiftests(9) + iiftests(1.111) + [cp.asin(9/10)] + + ret_test = (c_i * 100 + 5,) + ret_ref = (9 * 100 + 5,) + + out = [Store(r) for r in ret_test] + + sdb = backend.stencil_db_from_package('armv7thumb') + dw, variables = compile_to_dag(out, sdb) + + #dw.write_com(_binwrite.Command.READ_DATA) + #dw.write_int(0) + #dw.write_int(28) + + # run program command + #dw.write_com(_binwrite.Command.RUN_PROG) + dw.write_com(_binwrite.Command.DUMP_CODE) + + for v in ret_test: + assert isinstance(v, value) + add_read_value_remote(dw, variables, v.net) + + #dw.write_com(_binwrite.Command.READ_DATA) + #dw.write_int(0) + #dw.write_int(28) + + dw.write_com(_binwrite.Command.END_COM) + + #print('* Data to runner:') + #dw.print() + + dw.to_file('build/runner/test-armv7thumb.copapy') + + if not check_for_qemu(): + warnings.warn("qemu-armv7 not found, armv7 test skipped!", UserWarning) + return + if not os.path.isfile('build/runner/coparun-armv7thumb'): + warnings.warn("armv7thumb runner not found, armv7thumb test skipped!", UserWarning) + return + + command = qemu_command + ['build/runner/coparun-armv7thumb', 'build/runner/test-armv7thumb.copapy'] + ['build/runner/test.copapy-armv7thumb.bin'] + #try: + result = run_command(command) + #except FileNotFoundError: + # warnings.warn(f"Test skipped, executable not found.", UserWarning) + # return + + print('* Output from runner:\n--') + print(result) + print('--') + + assert 'Return value: 1' in result + + result_data = parse_results(result) + + for test, ref in zip(ret_test, ret_ref): + assert isinstance(test, value) + address = variables[test.net][0] + data = result_data[address] + if test.dtype == 'int': + val = int.from_bytes(data, sdb.byteorder, signed=True) + elif test.dtype == 'bool': + val = bool.from_bytes(data, sdb.byteorder) + elif test.dtype == 'float': + en = {'little': '<', 'big': '>'}[sdb.byteorder] + val = struct.unpack(en + 'f', data)[0] + assert isinstance(val, float) + else: + raise Exception(f"Unknown type: {test.dtype}") + print('+', val, ref, test.dtype, f" addr={address}") + for t in (int, float, bool): + assert isinstance(val, t) == isinstance(ref, t), f"Result type does not match for {val} and {ref}" + assert val == pytest.approx(ref, 1e-5), f"Result does not match: {val} and reference: {ref}" # pyright: ignore[reportUnknownMemberType] + + +if __name__ == "__main__": + #test_example() + test_slow_31bit_int_list_hash() diff --git a/tools/build.sh b/tools/build.sh index b7e3cd6..fca6f7e 100644 --- a/tools/build.sh +++ b/tools/build.sh @@ -4,10 +4,10 @@ set -eux ARCH=${1:-x86_64} case "$ARCH" in - (x86_64|arm-v6|arm-v7|all) + (x86_64|arm-v6|arm-v7|arm-v7-thumb|all) ;; (*) - echo "Usage: $0 [x86_64|arm-v6|arm-v7|all]" + echo "Usage: $0 [x86_64|arm-v6|arm-v7|arm-v6-thumb|all]" exit 1 ;; esac @@ -107,3 +107,36 @@ if [[ "$ARCH" == "arm-v7" || "$ARCH" == "all" ]]; then src/coparun/mem_man.c \ -o build/runner/coparun-armv7 fi + +####################################### +# ARM v7 thumb +####################################### +if [[ "$ARCH" == "arm-v7-thumb" || "$ARCH" == "all" ]]; then + echo "--------------arm-v7-thumb 32 bit----------------" + + LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) + + arm-none-eabi-gcc -fno-pic -ffunction-sections \ + -march=armv7e-m -mfpu=fpv4-sp-d16 -mfloat-abi=hard -mthumb \ + -c $SRC -O3 -o build/stencils/stencils.o + + arm-none-eabi-ld -r \ + build/stencils/stencils.o \ + build/musl/musl_objects_armv7thumb.o \ + $LIBGCC \ + -o $DEST/stencils_armv7thumb_O3.o + + arm-none-eabi-objdump -d -x \ + $DEST/stencils_armv7thumb_O3.o \ + > build/stencils/stencils_armv7thumb_O3.asm + + arm-linux-gnueabihf-gcc \ + -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb -static \ + -Wall -Wextra -Wconversion -Wsign-conversion \ + -Wshadow -Wstrict-overflow -O3 \ + -DENABLE_LOGGING \ + src/coparun/runmem.c \ + src/coparun/coparun.c \ + src/coparun/mem_man.c \ + -o build/runner/coparun-armv7thumb +fi \ No newline at end of file diff --git a/tools/inspect.sh b/tools/inspect.sh index c25e898..42fba43 100644 --- a/tools/inspect.sh +++ b/tools/inspect.sh @@ -14,3 +14,5 @@ objdump -D -b binary -m i386:x86-64 --adjust-vma=0x1000 build/runner/test.copapy build/runner/coparun-armv7 build/runner/test-armv7.copapy build/runner/test.copapy-armv7.bin arm-none-eabi-objdump -D -b binary -marm --adjust-vma=0x50000 build/runner/test.copapy-armv7.bin > build/runner/test.copapy-armv7.asm + +# arm-none-eabi-objdump -D -b binary -marm -M force-thumb --adjust-vma=0x50001 build/runner/test.copapy-armv7thumb.bin > build/runner/test.copapy-armv7thumb.asm From e52cbe9e1b425462ae81296700cafce25d345818 Mon Sep 17 00:00:00 2001 From: Nicolas Kruse Date: Thu, 5 Feb 2026 14:26:07 +0100 Subject: [PATCH 05/27] docstrings added and descriptions for c files added --- src/copapy/_basic_types.py | 2 +- src/copapy/_compiler.py | 4 ++-- src/copapy/_stencils.py | 14 +++++++++++--- src/coparun/coparun.c | 12 ++++++++++++ src/coparun/coparun_module.c | 7 +++++++ src/coparun/mem_man.c | 8 ++++++++ src/coparun/runmem.c | 7 +++++++ src/coparun/runmem.h | 7 +++++++ 8 files changed, 55 insertions(+), 6 deletions(-) diff --git a/src/copapy/_basic_types.py b/src/copapy/_basic_types.py index 0a120b2..eb412f1 100644 --- a/src/copapy/_basic_types.py +++ b/src/copapy/_basic_types.py @@ -429,8 +429,8 @@ class Op(Node): def __hash__(self) -> int: return self.node_hash -# Interface for vector and tensor types class ArrayType(Generic[TNum]): + """Interface for vector and tensor types.""" def __init__(self, shape: tuple[int, ...]) -> None: self.shape = shape self.values: tuple[TNum | value[TNum], ...] = () diff --git a/src/copapy/_compiler.py b/src/copapy/_compiler.py index cbade28..929b265 100644 --- a/src/copapy/_compiler.py +++ b/src/copapy/_compiler.py @@ -134,7 +134,7 @@ def get_const_nets(nodes: list[Node]) -> list[Net]: def add_load_ops(node_list: list[Node]) -> Generator[tuple[Net | None, Node], None, None]: - """Add read node before each op where arguments are not already positioned + """Add load/read node before each op where arguments are not already positioned correctly in the registers Arguments: @@ -172,7 +172,7 @@ def add_load_ops(node_list: list[Node]) -> Generator[tuple[Net | None, Node], No def add_store_ops(net_node_list: list[tuple[Net | None, Node]], const_nets: list[Net]) -> Generator[tuple[Net | None, Node], None, None]: - """Add write operation for each new defined net if a read operation is later followed + """Add store/write operation for each new defined net if a read operation is later followed Returns: Yields tuples of a net and a node. The associated net is provided for read and write nodes. diff --git a/src/copapy/_stencils.py b/src/copapy/_stencils.py index b7864c0..b746c94 100644 --- a/src/copapy/_stencils.py +++ b/src/copapy/_stencils.py @@ -32,11 +32,14 @@ class relocation_entry: @dataclass class patch_entry: """ - A dataclass for representing a relocation entry + A dataclass for representing a patch entry Attributes: - addr (int): address of first byte to patch relative to the start of the symbol - type (RelocationType): relocation type + mask (int): Bit-mask to apply to the patched value + address (int): Address where to patch + value (int): The value to write at the patch address + scale (int): The scale factor for the patch value + patch_type (int): The type of patch """ mask: int address: int @@ -46,6 +49,11 @@ class patch_entry: def detect_process_arch() -> str: + """ + For running the code locally in the python module + the architecture of the current process is detected + by this function to load the correct stencil database. + """ bits = struct.calcsize("P") * 8 arch = platform.machine().lower() diff --git a/src/coparun/coparun.c b/src/coparun/coparun.c index 45da6d3..10dfada 100644 --- a/src/coparun/coparun.c +++ b/src/coparun/coparun.c @@ -1,3 +1,15 @@ +/* + * file: coparun.c + * Description: This file alows to run copapy programs in the command line + * reading copapy data, code and patch instructions from a file and jump to the + * entry point to execute it or dump the patched code memory for debugging to a file. + * + * It's intended for testing and debugging purposes, to run copapy programs in a + * debugger or emulator like qemu. + * + * Usage: coparun [memory_dump_file] + */ + #include #include #include "runmem.h" diff --git a/src/coparun/coparun_module.c b/src/coparun/coparun_module.c index bee0dfb..1e50f29 100644 --- a/src/coparun/coparun_module.c +++ b/src/coparun/coparun_module.c @@ -1,3 +1,10 @@ +/* + * file: coparun_module.c + * Description: This file defines a Python C extension module that provides an + * interface to the core functions of the coparun runner, to run copapy programs + * directly local in Python. + */ + #define PY_SSIZE_T_CLEAN #include #include "runmem.h" diff --git a/src/coparun/mem_man.c b/src/coparun/mem_man.c index 0a5bdb5..8e89978 100644 --- a/src/coparun/mem_man.c +++ b/src/coparun/mem_man.c @@ -1,3 +1,11 @@ +/* + * file: mem_man.c + * Description: This file contains memory management functions for the coparun + * runner, including allocation and deallocation of executable and data memory. + * Depending of the target operating system or bare metal environment, it + * handles memory management accordingly. +*/ + #include #include #include diff --git a/src/coparun/runmem.c b/src/coparun/runmem.c index 1165956..f6e54b3 100644 --- a/src/coparun/runmem.c +++ b/src/coparun/runmem.c @@ -1,3 +1,10 @@ +/* + * file: runmem.c + * Description: This file contain the core functions of the runner + * to receive data, code and patch instruction, does the patching + * and jumps to the entry point of the copapy program + */ + #include #include #include diff --git a/src/coparun/runmem.h b/src/coparun/runmem.h index 31e9905..77fe5d2 100644 --- a/src/coparun/runmem.h +++ b/src/coparun/runmem.h @@ -1,3 +1,10 @@ +/** + * @file runmem.h + * @brief Header file for runmem.c, which contains core functions of + * the runner to receive data, code and patch instructions, perform + * patching, and jump to the entry point of the copapy program. + */ + #ifndef RUNMEM_H #define RUNMEM_H From bc0ccd90b746e7cf49437fead765fa272d6011c8 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 12 Feb 2026 23:54:20 +0100 Subject: [PATCH 06/27] Updated pelfy and using offset_in_section instead of fields['st_value'] --- src/copapy/_stencils.py | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/src/copapy/_stencils.py b/src/copapy/_stencils.py index f464b3e..1c3c9ef 100644 --- a/src/copapy/_stencils.py +++ b/src/copapy/_stencils.py @@ -102,7 +102,7 @@ def get_last_call_in_function(func: pelfy.elf_symbol) -> int: instruction_lengths = 4 if reloc.bits < 32 else 5 address_field_length = 4 #print(f"-> {[r.fields['r_offset'] - func.fields['st_value'] for r in func.relocations]}") - return reloc.fields['r_offset'] - func.fields['st_value'] + address_field_length - instruction_lengths + return reloc.fields['r_offset'] - func.offset_in_section + address_field_length - instruction_lengths def get_op_after_last_call_in_function(func: pelfy.elf_symbol) -> int: @@ -110,7 +110,7 @@ def get_op_after_last_call_in_function(func: pelfy.elf_symbol) -> int: assert func.relocations, f'No call function in stencil function {func.name}.' reloc = func.relocations[-1] assert reloc.bits <= 32, "Relocation segment might be larger then 32 bit" - return reloc.fields['r_offset'] - func.fields['st_value'] + 4 + return reloc.fields['r_offset'] - func.offset_in_section + 4 class stencil_database(): @@ -121,7 +121,7 @@ class stencil_database(): var_size (dict[str, int]): dictionary of object names and their sizes byteorder (ByteOrder): byte order of the ELF file elf (elf_file): the loaded ELF file - thumb_mode (int): 1 if ARM in thumb mode, 0 otherwise + thumb_mode (bool): entry_function_shell in ARM thumb mode """ def __init__(self, obj_file: str | bytes): @@ -148,10 +148,7 @@ class stencil_database(): # if s.info == 'STT_OBJECT'} self.byteorder: ByteOrder = self.elf.byteorder - self.arm = '.ARM.attributes' in self.elf.sections - - # Returns 1 for ARM in thumb mode, 0 otherwise - self.thumb_mode = self.elf.symbols['entry_function_shell'].fields['st_value'] & 1 + self.thumb_mode = self.elf.symbols['entry_function_shell'].thumb_mode #for name in self.function_definitions.keys(): # sym = self.elf.symbols[name] @@ -194,14 +191,14 @@ class stencil_database(): for reloc in symbol.relocations: # address to fist byte to patch relative to the start of the symbol - patch_offset = reloc.fields['r_offset'] - symbol.fields['st_value'] - start_index + patch_offset = reloc.fields['r_offset'] - symbol.offset_in_section - start_index if patch_offset < end_index - start_index: # Exclude the call to the result_* function reloc_entry = relocation_entry(reloc.symbol.name, reloc.symbol.info, - reloc.symbol.fields['st_value'], + reloc.symbol.fields['st_value'], # LSB on ARM indicates thumb mode reloc.symbol.fields['st_shndx'], - symbol.fields['st_value'], + symbol.offset_in_section, start_index, reloc) cache.append(reloc_entry) @@ -348,10 +345,7 @@ class stencil_database(): start_stencil, end_stencil = get_stencil_position(func) assert func.section - # For arm functions, mask out the thumb mode bit - function_offset = func.fields['st_value'] & ~int(self.arm) - - start_index = func.section['sh_offset'] + function_offset + start_stencil + start_index = func.offset_in_file + start_stencil lengths = end_stencil - start_stencil self._stencil_cache[name] = (start_index, lengths) @@ -389,7 +383,7 @@ class stencil_database(): def get_symbol_offset(self, name: str) -> int: """Returns the offset of a specified symbol in the section.""" - return self.elf.symbols[name].fields['st_value'] + return self.elf.symbols[name].offset_in_section def get_symbol_section_index(self, name: str) -> int: """Returns the section index for a specified symbol name.""" @@ -420,13 +414,6 @@ class stencil_database(): func = self.elf.symbols[name] assert func.info == 'STT_FUNC', f"{name} is not a function" - # For arm functions, mask out the thumb mode bit - function_offset = func.fields['st_value'] & ~int(self.arm) - - start_index = func.section['sh_offset'] + function_offset - lengths = end_stencil - start_stencil - - #return self.elf.read_bytes(start_index, func.fields['st_size']) if part == 'start': index = get_last_call_in_function(func) return func.data[:index] From a81236a3fca47eafcc94e7ac6b8c93249a42ecf1 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Fri, 13 Feb 2026 00:51:55 +0100 Subject: [PATCH 07/27] 4-Byte-Alignment error on ARM thumb fixed by using section size instead of function size to include nop padding --- src/copapy/_stencils.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/copapy/_stencils.py b/src/copapy/_stencils.py index 1c3c9ef..eb81aba 100644 --- a/src/copapy/_stencils.py +++ b/src/copapy/_stencils.py @@ -80,11 +80,20 @@ def get_return_function_type(symbol: pelfy.elf_symbol) -> str: def get_stencil_position(func: pelfy.elf_symbol) -> tuple[int, int]: start_index = 0 # There must be no prolog + # Find last relocation in function last_instr = get_last_call_in_function(func) - function_size = func.fields['st_size'] - if last_instr + 5 >= function_size: # Check if jump is last instruction - end_index = last_instr # Jump can be striped + + assert func.section, f"No code section specified for symbol {func.name}" + + # func.section.fields['sh_size'] is equivalent to func.fields['st_size'] + # expect for ARM thumb, here nop padding at the end for 4-byte alignment + # is not included in st_size + function_size = func.section.fields['sh_size'] + + # Check if jump is the last instruction and can be striped + if last_instr + 5 >= function_size: + end_index = last_instr else: end_index = function_size @@ -98,7 +107,8 @@ def get_last_call_in_function(func: pelfy.elf_symbol) -> int: if reloc.symbol.name.startswith('dummy_'): return -0xFFFF # Last relocation is not a jump else: - # Assume the call instruction is 4 bytes long for relocations with less than 32 bit and 5 bytes otherwise + # Assume the jump/call instruction is 4 bytes long for relocations + # with less than 32 bit and 5 bytes otherwise instruction_lengths = 4 if reloc.bits < 32 else 5 address_field_length = 4 #print(f"-> {[r.fields['r_offset'] - func.fields['st_value'] for r in func.relocations]}") From 83ce6ce0e7f6bdf518f999351e3c877be0f55e84 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Fri, 13 Feb 2026 01:13:01 +0100 Subject: [PATCH 08/27] musl functions for math on ARM thumb added to stencil build pipeline --- tools/cross_compiler_unix/packobjs.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/cross_compiler_unix/packobjs.sh b/tools/cross_compiler_unix/packobjs.sh index 43bd0e9..ff5f32a 100644 --- a/tools/cross_compiler_unix/packobjs.sh +++ b/tools/cross_compiler_unix/packobjs.sh @@ -25,14 +25,16 @@ ar x ../../musl/lib/libc.a sinf.o cosf.o tanf.o asinf.o acosf.o atanf.o atan2f.o ar x ../../musl/lib/libc.a sqrtf.o logf.o expf.o sqrt.o ar x ../../musl/lib/libc.a logf_data.o __tandf.o __cosdf.o __sindf.o ar x ../../musl/lib/libc.a fabsf.o scalbn.o floor.o floorf.o exp2f_data.o powf.o powf_data.o -ar x ../../musl/lib/libc.a __rem_pio2f.o __math_invalidf.o __stack_chk_fail.o __math_divzerof.o __math_oflowf.o __rem_pio2_large.o __math_uflowf.o __math_xflowf.o +ar x ../../musl/lib/libc.a __rem_pio2f.o __math_invalid.o __math_invalidf.o __stack_chk_fail.o +ar x ../../musl/lib/libc.a __math_divzerof.o __math_oflowf.o __rem_pio2_large.o __math_uflowf.o __math_xflowf.o __rsqrt_tab.o # Check out .lo (PIC) ar x ../../musl/lib/libc.a sinf.lo cosf.lo tanf.lo asinf.lo acosf.lo atanf.lo atan2f.lo ar x ../../musl/lib/libc.a sqrtf.lo logf.lo expf.lo sqrt.lo ar x ../../musl/lib/libc.a logf_data.lo __tandf.lo __cosdf.lo __sindf.lo ar x ../../musl/lib/libc.a fabsf.lo scalbn.lo floor.lo floorf.o exp2f_data.lo powf.lo powf_data.lo -ar x ../../musl/lib/libc.a __rem_pio2f.lo __math_invalidf.lo __stack_chk_fail.lo __math_divzerof.lo __math_oflowf.lo __rem_pio2_large.lo __math_uflowf.lo __math_xflowf.lo +ar x ../../musl/lib/libc.a __rem_pio2f.lo __math_invalid.lo __math_invalidf.lo __stack_chk_fail.lo +ar x ../../musl/lib/libc.a __math_divzerof.lo __math_oflowf.lo __rem_pio2_large.lo __math_uflowf.lo __math_xflowf.lo __rsqrt_tab.lo cd ../../musl From cabfda4ec6977a71e3a99b4dc613381a88c6ae0a Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sat, 28 Feb 2026 22:07:27 +0100 Subject: [PATCH 09/27] CI: stencil build script updated with Cortex-A Thumb version --- tools/cross_compiler_unix/build_musl.sh | 5 ++++- tools/cross_compiler_unix/packobjs.sh | 4 ++-- tools/crosscompile.sh | 9 +++++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tools/cross_compiler_unix/build_musl.sh b/tools/cross_compiler_unix/build_musl.sh index 2f16b1e..3f7b50e 100644 --- a/tools/cross_compiler_unix/build_musl.sh +++ b/tools/cross_compiler_unix/build_musl.sh @@ -26,8 +26,11 @@ sh ../packobjs.sh arm-none-eabi-gcc arm-none-eabi-ld /object_files/musl_objects_ # Armv7 sh ../packobjs.sh arm-none-eabi-gcc arm-none-eabi-ld /object_files/musl_objects_armv7.o "-march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm" +# Armv7 Thumb for Cortex-A +sh ../packobjs.sh arm-none-eabi-gcc arm-none-eabi-ld /object_files/musl_objects_armv7thumb.o "-march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb" + # Armv7 Thumb for Cortex-M3..7 -sh ../packobjs.sh arm-none-eabi-gcc arm-none-eabi-ld /object_files/musl_objects_armv7thumb.o "-march=armv7e-m -mfpu=fpv4-sp-d16 -mfloat-abi=hard -mthumb" +sh ../packobjs.sh arm-none-eabi-gcc arm-none-eabi-ld /object_files/musl_objects_armv7mthumb.o "-march=armv7e-m -mfpu=fpv4-sp-d16 -mfloat-abi=hard -mthumb" #sh ../packobjs.sh mips mips-linux-gnu-gcc-13 mips-linux-gnu-ld diff --git a/tools/cross_compiler_unix/packobjs.sh b/tools/cross_compiler_unix/packobjs.sh index ff5f32a..98e3c17 100644 --- a/tools/cross_compiler_unix/packobjs.sh +++ b/tools/cross_compiler_unix/packobjs.sh @@ -26,7 +26,7 @@ ar x ../../musl/lib/libc.a sqrtf.o logf.o expf.o sqrt.o ar x ../../musl/lib/libc.a logf_data.o __tandf.o __cosdf.o __sindf.o ar x ../../musl/lib/libc.a fabsf.o scalbn.o floor.o floorf.o exp2f_data.o powf.o powf_data.o ar x ../../musl/lib/libc.a __rem_pio2f.o __math_invalid.o __math_invalidf.o __stack_chk_fail.o -ar x ../../musl/lib/libc.a __math_divzerof.o __math_oflowf.o __rem_pio2_large.o __math_uflowf.o __math_xflowf.o __rsqrt_tab.o +ar x ../../musl/lib/libc.a __math_divzerof.o __math_oflowf.o __rem_pio2_large.o __math_uflowf.o __math_xflowf.o sqrt_data.o # Check out .lo (PIC) ar x ../../musl/lib/libc.a sinf.lo cosf.lo tanf.lo asinf.lo acosf.lo atanf.lo atan2f.lo @@ -34,7 +34,7 @@ ar x ../../musl/lib/libc.a sqrtf.lo logf.lo expf.lo sqrt.lo ar x ../../musl/lib/libc.a logf_data.lo __tandf.lo __cosdf.lo __sindf.lo ar x ../../musl/lib/libc.a fabsf.lo scalbn.lo floor.lo floorf.o exp2f_data.lo powf.lo powf_data.lo ar x ../../musl/lib/libc.a __rem_pio2f.lo __math_invalid.lo __math_invalidf.lo __stack_chk_fail.lo -ar x ../../musl/lib/libc.a __math_divzerof.lo __math_oflowf.lo __rem_pio2_large.lo __math_uflowf.lo __math_xflowf.lo __rsqrt_tab.lo +ar x ../../musl/lib/libc.a __math_divzerof.lo __math_oflowf.lo __rem_pio2_large.lo __math_uflowf.lo __math_xflowf.lo sqrt_data.lo cd ../../musl diff --git a/tools/crosscompile.sh b/tools/crosscompile.sh index 9494ff2..29a15ca 100644 --- a/tools/crosscompile.sh +++ b/tools/crosscompile.sh @@ -36,15 +36,20 @@ arm-none-eabi-gcc -march=armv6 -mfpu=vfp -mfloat-abi=hard -marm $FLAGS -$OPT -c LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) arm-none-eabi-ld -r $STMP /object_files/musl_objects_armv6.o $LIBGCC -o $DEST/stencils_armv6_$OPT.o -# ARMv7 hardware fp +# ARMv7 hardware fp for Cortex-A arm-none-eabi-gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm $FLAGS -$OPT -c $SRC -o $STMP LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) arm-none-eabi-ld -r $STMP /object_files/musl_objects_armv7.o $LIBGCC -o $DEST/stencils_armv7_$OPT.o +# ARMv7 Thumb for Cortex-A with hardware fp +arm-none-eabi-gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb $FLAGS -$OPT -c $SRC -o $STMP +LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) +arm-none-eabi-ld -r $STMP /object_files/musl_objects_armv7thumb.o $LIBGCC -o $DEST/stencils_armv7thumb_$OPT.o + # Armv7 Thumb for Cortex-M3..7 hardware fp arm-none-eabi-gcc -march=armv7e-m -mfpu=fpv4-sp-d16 -mfloat-abi=hard -mthumb $FLAGS -$OPT -c $SRC -o $STMP LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) -arm-none-eabi-ld -r $STMP /object_files/musl_objects_armv7thumb.o $LIBGCC -o $DEST/stencils_armv7thumb_$OPT.o +arm-none-eabi-ld -r $STMP /object_files/musl_objects_armv7mthumb.o $LIBGCC -o $DEST/stencils_armv7mthumb_$OPT.o # PowerPC64LE # powerpc64le-linux-gnu-gcc-13 $FLAGS -$OPT -c $SRC -o $DEST/stencils_ppc64le_$OPT.o From 436a09c1eac61cbab068520c76066063cdc460e5 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sat, 28 Feb 2026 22:08:22 +0100 Subject: [PATCH 10/27] copy method to data_writer added --- src/copapy/_binwrite.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/copapy/_binwrite.py b/src/copapy/_binwrite.py index a12d82b..e1d7c90 100644 --- a/src/copapy/_binwrite.py +++ b/src/copapy/_binwrite.py @@ -24,6 +24,11 @@ class data_writer(): self._data: list[tuple[str, bytes, int]] = [] self.byteorder: ByteOrder = byteorder + def copy(self): + cp = data_writer(self.byteorder) + cp._data = self._data.copy() + return cp + def write_int(self, value: int, num_bytes: int = 4, signed: bool = False) -> None: self._data.append((f"INT {value}", value.to_bytes(length=num_bytes, byteorder=self.byteorder, signed=signed), 0)) From afc442ada62053d16e78b0cec6f4d34237907f45 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Sat, 28 Feb 2026 22:09:12 +0100 Subject: [PATCH 11/27] stencil build script updated --- tools/build.sh | 47 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/tools/build.sh b/tools/build.sh index fca6f7e..4af186a 100644 --- a/tools/build.sh +++ b/tools/build.sh @@ -1,13 +1,13 @@ #!/bin/bash -set -eux +set -eu ARCH=${1:-x86_64} case "$ARCH" in - (x86_64|arm-v6|arm-v7|arm-v7-thumb|all) + (x86_64|arm-v6|arm-v7|arm-v7-thumb|arm-v7m-thumb|all) ;; (*) - echo "Usage: $0 [x86_64|arm-v6|arm-v7|arm-v6-thumb|all]" + echo "Usage: $0 [x86_64|arm-v6|arm-v7|arm-v6-thumb|arm-v7m-thumb|all]" exit 1 ;; esac @@ -109,20 +109,20 @@ if [[ "$ARCH" == "arm-v7" || "$ARCH" == "all" ]]; then fi ####################################### -# ARM v7 thumb +# ARM v7 thumb Cortex-A ####################################### if [[ "$ARCH" == "arm-v7-thumb" || "$ARCH" == "all" ]]; then - echo "--------------arm-v7-thumb 32 bit----------------" + echo "--------------arm-v7a-thumb 32 bit----------------" LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) arm-none-eabi-gcc -fno-pic -ffunction-sections \ - -march=armv7e-m -mfpu=fpv4-sp-d16 -mfloat-abi=hard -mthumb \ + -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb \ -c $SRC -O3 -o build/stencils/stencils.o arm-none-eabi-ld -r \ build/stencils/stencils.o \ - build/musl/musl_objects_armv7thumb.o \ + build/musl/musl_objects_armv7.o \ $LIBGCC \ -o $DEST/stencils_armv7thumb_O3.o @@ -130,6 +130,39 @@ if [[ "$ARCH" == "arm-v7-thumb" || "$ARCH" == "all" ]]; then $DEST/stencils_armv7thumb_O3.o \ > build/stencils/stencils_armv7thumb_O3.asm + arm-linux-gnueabihf-gcc \ + -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb -static \ + -Wall -Wextra -Wconversion -Wsign-conversion \ + -Wshadow -Wstrict-overflow -O3 \ + -DENABLE_LOGGING \ + src/coparun/runmem.c \ + src/coparun/coparun.c \ + src/coparun/mem_man.c \ + -o build/runner/coparun-armv7thumb +fi + +####################################### +# ARM v7 thumb Cortex-M +####################################### +if [[ "$ARCH" == "arm-v7m-thumb" || "$ARCH" == "all" ]]; then + echo "--------------arm-v7m-thumb 32 bit----------------" + + LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) + + arm-none-eabi-gcc -fno-pic -ffunction-sections \ + -march=armv7e-m -mfpu=fpv4-sp-d16 -mfloat-abi=hard -mthumb \ + -c $SRC -O3 -o build/stencils/stencils.o + + arm-none-eabi-ld -r \ + build/stencils/stencils.o \ + build/musl/musl_objects_armv7mthumb.o \ + $LIBGCC \ + -o $DEST/stencils_armv7mthumb_O3.o + + arm-none-eabi-objdump -d -x \ + $DEST/stencils_armv7mthumb_O3.o \ + > build/stencils/stencils_armv7mthumb_O3.asm + arm-linux-gnueabihf-gcc \ -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb -static \ -Wall -Wextra -Wconversion -Wsign-conversion \ From 8fcf0dedac588777df63e1fcbbcb3cc3fd2814d1 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 2 Mar 2026 21:28:05 +0100 Subject: [PATCH 12/27] patch type added: PATCH_OBJECT_ARM32_ABS_THM (for R_ARM_THM_MOVW_ABS_NC and R_ARM_THM_MOVT_ABS) --- src/copapy/_binwrite.py | 1 + src/coparun/runmem.c | 40 ++++++++++++++++++++++++++++++++++++++-- src/coparun/runmem.h | 1 + 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/copapy/_binwrite.py b/src/copapy/_binwrite.py index e1d7c90..1af2ee3 100644 --- a/src/copapy/_binwrite.py +++ b/src/copapy/_binwrite.py @@ -13,6 +13,7 @@ Command = Enum('Command', [('ALLOCATE_DATA', 1), ('COPY_DATA', 2), ('PATCH_OBJECT_ABS', 0x2002), ('PATCH_OBJECT_REL', 0x2003), ('PATCH_OBJECT_ARM32_ABS', 0x2004), + ('PATCH_OBJECT_ARM32_ABS_THM', 0x2006), ('ENTRY_POINT', 7), ('RUN_PROG', 64), ('READ_DATA', 65), ('END_COM', 256), ('FREE_MEMORY', 257), ('DUMP_CODE', 258)]) diff --git a/src/coparun/runmem.c b/src/coparun/runmem.c index e5180e5..b905e97 100644 --- a/src/coparun/runmem.c +++ b/src/coparun/runmem.c @@ -50,6 +50,32 @@ void patch_arm32_abs(uint8_t *patch_addr, uint32_t imm16) *((uint32_t *)patch_addr) = instr; } +void patch_arm_thm_abs(uint8_t *patch_addr, uint32_t imm16) +{ + // Thumb MOVW (T3) / MOVT (T1) encoding + + uint16_t *instr16 = (uint16_t *)patch_addr; + uint16_t first_half = instr16[0]; + uint16_t second_half = instr16[1]; + + // Extract fields from imm16 + uint32_t imm4 = (imm16 >> 12) & 0xF; + uint32_t i = (imm16 >> 11) & 0x1; + uint32_t imm3 = (imm16 >> 8) & 0x7; + uint32_t imm8 = imm16 & 0xFF; + + // Clear bits + first_half &= (uint16_t)(~(0x000F | (1 << 10))); + second_half &= (uint16_t)(~(0x00FF | (0x7 << 12))); + + // Set new fields + first_half |= (uint16_t)((imm4 << 0) | (i << 10)); + second_half |= (uint16_t)(imm8 | (imm3 << 12)); + + instr16[0] = first_half; + instr16[1] = second_half; +} + void patch_arm_thm_jump24(uint8_t *patch_addr, int32_t imm24) { // Read the 32-bit instruction (two halfwords) @@ -77,8 +103,8 @@ void patch_arm_thm_jump24(uint8_t *patch_addr, int32_t imm24) second_half &= 0xD000; // Keep upper 5 bits // Set new imm fields - first_half |= (S << 10) | imm10; - second_half |= (J1 << 13) | (J2 << 11) | imm11; + first_half |= (uint16_t)((S << 10) | imm10); + second_half |= (uint16_t)((J1 << 13) | (J2 << 11) | imm11); // Write back instr16[0] = first_half; @@ -225,6 +251,16 @@ int parse_commands(runmem_t *context, uint8_t *bytes) { patch_arm_thm_jump24(context->executable_memory + offs, value); break; + case PATCH_OBJECT_ARM32_ABS_THM: + offs = *(uint32_t*)bytes; bytes += 4; + patch_mask = *(uint32_t*)bytes; bytes += 4; + patch_scale = *(int32_t*)bytes; bytes += 4; + value = *(int32_t*)bytes; bytes += 4; + LOG("PATCH_OBJECT_ARM32_ABS_THM patch_offs=%i patch_mask=%#08x scale=%i value=%i imm16=%#04x\n", + offs, patch_mask, patch_scale, value, (uint32_t)((uintptr_t)(context->data_memory + value) & patch_mask) / (uint32_t)patch_scale); + patch_arm_thm_abs(context->executable_memory + offs, (uint32_t)((uintptr_t)(context->data_memory + value) & patch_mask) / (uint32_t)patch_scale); + break; + case ENTRY_POINT: rel_entr_point = *(uint32_t*)bytes; bytes += 4; context->entr_point = (entry_point_t)(context->executable_memory + rel_entr_point); diff --git a/src/coparun/runmem.h b/src/coparun/runmem.h index 371f2f3..17da047 100644 --- a/src/coparun/runmem.h +++ b/src/coparun/runmem.h @@ -26,6 +26,7 @@ #define PATCH_OBJECT_ABS 0x2002 #define PATCH_OBJECT_REL 0x2003 #define PATCH_OBJECT_ARM32_ABS 0x2004 +#define PATCH_OBJECT_ARM32_ABS_THM 0x2006 #define ENTRY_POINT 7 #define RUN_PROG 64 #define READ_DATA 65 From c7c8db633230abf7defb107a899c74d56973cf62 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 2 Mar 2026 21:28:46 +0100 Subject: [PATCH 13/27] R_ARM_THM_MOV* support added --- src/copapy/_compiler.py | 6 +++--- src/copapy/_stencils.py | 31 +++++++++++++++++++++++++------ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/copapy/_compiler.py b/src/copapy/_compiler.py index cd42d41..156fbe9 100644 --- a/src/copapy/_compiler.py +++ b/src/copapy/_compiler.py @@ -393,14 +393,14 @@ def compile_to_dag(node_list: Iterable[Node], sdb: stencil_database) -> tuple[bi # assemble stencils to main program and patch stencils data = sdb.get_function_code('entry_function_shell', 'start') data_list.append(data) - print(f"* entry_function_shell (0) " + ' '.join(f'{d:02X}' for d in data)) + #print(f"* entry_function_shell (0) " + ' '.join(f'{d:02X}' for d in data)) offset = aux_func_len + len(data) for associated_net, node in extended_output_ops: assert node.name in sdb.stencil_definitions, f"- Warning: {node.name} stencil not found" data = sdb.get_stencil_code(node.name) data_list.append(data) - print(f"* {node.name} ({offset}) " + ' '.join(f'{d:02X}' for d in data)) + #print(f"* {node.name} ({offset}) " + ' '.join(f'{d:02X}' for d in data)) for reloc in sdb.get_relocations(node.name, stencil=True): if reloc.target_symbol_info in ('STT_OBJECT', 'STT_NOTYPE', 'STT_SECTION'): @@ -453,7 +453,7 @@ def compile_to_dag(node_list: Iterable[Node], sdb: stencil_database) -> tuple[bi for reloc in sdb.get_relocations(name): if not reloc.target_section_index: - assert reloc.pelfy_reloc.type == 'R_ARM_V4BX', (reloc.pelfy_reloc.type, name) + assert reloc.pelfy_reloc.type == 'R_ARM_V4BX', (reloc.pelfy_reloc.type, name, reloc.pelfy_reloc.symbol.name) elif reloc.target_symbol_info in {'STT_OBJECT', 'STT_NOTYPE', 'STT_SECTION'}: # Patch constants/variable addresses on heap diff --git a/src/copapy/_stencils.py b/src/copapy/_stencils.py index eb81aba..a317ea6 100644 --- a/src/copapy/_stencils.py +++ b/src/copapy/_stencils.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from typing import Generator, Literal, Iterable, TYPE_CHECKING import struct import platform +import os if TYPE_CHECKING: import pelfy @@ -46,6 +47,10 @@ class patch_entry: def detect_process_arch() -> str: + cp_target_arch = os.environ.get("CP_TARGET_ARCH") + if cp_target_arch: + return cp_target_arch + bits = struct.calcsize("P") * 8 arch = platform.machine().lower() @@ -305,21 +310,20 @@ class stencil_database(): scale = 8 #print(f" *> {patch_value=} {symbol_address=} {pr.fields['r_addend']=}, {function_offset=}") - elif pr.type.endswith('_MOVW_ABS_NC'): - # R_ARM_MOVW_ABS_NC + elif pr.type == 'R_ARM_MOVW_ABS_NC': # (S + A) & 0xFFFF mask = 0xFFFF patch_value = symbol_address + pr.fields['r_addend'] symbol_type = symbol_type + 0x04 # Absolut value #print(f" *> {pr.type} {patch_value=} {symbol_address=}, {function_offset=}") - elif pr.type.endswith('_MOVT_ABS'): - # R_ARM_MOVT_ABS + elif pr.type =='R_ARM_MOVT_ABS': # (S + A) & 0xFFFF0000 mask = 0xFFFF0000 patch_value = symbol_address + pr.fields['r_addend'] symbol_type = symbol_type + 0x04 # Absolut value scale = 0x10000 + #print(f" *> {pr.type} {patch_value=} {symbol_address=}, {function_offset=}, {pr.fields['r_addend']=}") elif pr.type.endswith('_ABS32'): # R_ARM_ABS32 @@ -330,9 +334,24 @@ class stencil_database(): elif pr.type.endswith('_THM_JUMP24') or pr.type.endswith('_THM_CALL'): # R_ARM_THM_JUMP24 # S + A - P - #assert pr.fields['r_addend'] == 0, pr.fields['r_addend'] - patch_value = symbol_address - (patch_offset + 4) #+ pr.fields['r_addend'] + patch_value = symbol_address - patch_offset + pr.fields['r_addend'] symbol_type = symbol_type + 0x05 # PATCH_FUNC_ARM32_THM + #print(f" *> {pr.type} {patch_value=} {symbol_address=} {pr.fields['r_addend']=} {pr.bits=}, {function_offset=} {patch_offset=}") + + elif pr.type == 'R_ARM_THM_MOVW_ABS_NC': + # (S + A) & 0xFFFF + mask = 0xFFFF + patch_value = symbol_address + pr.fields['r_addend'] + symbol_type = symbol_type + 0x06 # PATCH_OBJECT_ARM32_ABS_THM + #print(f" *> {pr.type} {patch_value=} {symbol_address=}, {function_offset=}, {pr.fields['r_addend']=}") + + elif pr.type == 'R_ARM_THM_MOVT_ABS': + # (S + A) & 0xFFFF0000 + mask = 0xFFFF0000 + patch_value = symbol_address + pr.fields['r_addend'] + symbol_type = symbol_type + 0x06 # PATCH_OBJECT_ARM32_ABS_THM + scale = 0x10000 + #print(f" *> {pr.type} {patch_value=} {symbol_address=}, {function_offset=}, {pr.fields['r_addend']=}") else: raise NotImplementedError(f"Relocation type {pr.type} in {relocation.pelfy_reloc.target_section.name} pointing to {relocation.pelfy_reloc.symbol.name} not implemented") From 2eb49cc2e5fddbb52bcd1d268e867ece07dc49cc Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 2 Mar 2026 21:29:46 +0100 Subject: [PATCH 14/27] test for ARM thumb updated --- ...s_armv7thunb.py => test_ops_armv7thumb.py} | 45 ++++++++++++------- 1 file changed, 29 insertions(+), 16 deletions(-) rename tests/{test_ops_armv7thunb.py => test_ops_armv7thumb.py} (77%) diff --git a/tests/test_ops_armv7thunb.py b/tests/test_ops_armv7thumb.py similarity index 77% rename from tests/test_ops_armv7thunb.py rename to tests/test_ops_armv7thumb.py index ac67e51..983510e 100644 --- a/tests/test_ops_armv7thunb.py +++ b/tests/test_ops_armv7thumb.py @@ -55,6 +55,9 @@ def function1(c1: NumLike) -> list[NumLike]: c1 + 4, c1 - 4, c1 > 2, c1 > 100, c1 < 4, c1 < 100] +def function1ex(c1: NumLike) -> list[NumLike]: + return [c1 // 4] + def function2(c1: NumLike) -> list[NumLike]: return [c1 * 4.44, c1 * -4.44] @@ -90,24 +93,21 @@ def test_compile(): c_f = value(1.111) c_b = value(True) - #ret_test = function1(c_i) + function1(c_f) + function2(c_i) + function2(c_f) + function3(c_i) + function4(c_i) + function5(c_b) + [value(9) % 2] + iiftests(c_i) + iiftests(c_f) + [cp.asin(c_i/10)] - #ret_ref = function1(9) + function1(1.111) + function2(9) + function2(1.111) + function3(9) + function4(9) + function5(True) + [9 % 2] + iiftests(9) + iiftests(1.111) + [cp.asin(9/10)] - - ret_test = (c_i * 100 + 5,) - ret_ref = (9 * 100 + 5,) + ret_test = function1(c_i) + function1(c_f) + function2(c_i) + function2(c_f) + function3(c_i) + function4(c_i) + function5(c_b) + [value(9) % 2] + iiftests(c_i) + iiftests(c_f) + [cp.asin(c_i/10)] + ret_ref = function1(9) + function1(1.111) + function2(9) + function2(1.111) + function3(9) + function4(9) + function5(True) + [9 % 2] + iiftests(9) + iiftests(1.111) + [cp.asin(9/10)] out = [Store(r) for r in ret_test] - sdb = backend.stencil_db_from_package('armv7thumb') + sdb = backend.stencil_db_from_package('armv7mthumb') dw, variables = compile_to_dag(out, sdb) #dw.write_com(_binwrite.Command.READ_DATA) #dw.write_int(0) #dw.write_int(28) - # run program command - #dw.write_com(_binwrite.Command.RUN_PROG) - dw.write_com(_binwrite.Command.DUMP_CODE) + du = dw.copy() + dw.write_com(_binwrite.Command.RUN_PROG) + du.write_com(_binwrite.Command.DUMP_CODE) for v in ret_test: assert isinstance(v, value) @@ -118,11 +118,13 @@ def test_compile(): #dw.write_int(28) dw.write_com(_binwrite.Command.END_COM) + du.write_com(_binwrite.Command.END_COM) #print('* Data to runner:') #dw.print() dw.to_file('build/runner/test-armv7thumb.copapy') + du.to_file('build/runner/test-armv7thumb-dump.copapy') if not check_for_qemu(): warnings.warn("qemu-armv7 not found, armv7 test skipped!", UserWarning) @@ -131,12 +133,14 @@ def test_compile(): warnings.warn("armv7thumb runner not found, armv7thumb test skipped!", UserWarning) return - command = qemu_command + ['build/runner/coparun-armv7thumb', 'build/runner/test-armv7thumb.copapy'] + ['build/runner/test.copapy-armv7thumb.bin'] - #try: + print('----- Dump code...') + command = qemu_command + ['build/runner/coparun-armv7thumb', 'build/runner/test-armv7thumb-dump.copapy', 'build/runner/test.copapy-armv7thumb.bin'] result = run_command(command) - #except FileNotFoundError: - # warnings.warn(f"Test skipped, executable not found.", UserWarning) - # return + + print('----- Run code...') + command = qemu_command + ['build/runner/coparun-armv7thumb', 'build/runner/test-armv7thumb.copapy'] + result = run_command(command) + print('* Output from runner:\n--') print(result) @@ -167,5 +171,14 @@ def test_compile(): if __name__ == "__main__": - #test_example() - test_slow_31bit_int_list_hash() + test_compile() + + +""" +qemu-arm -d in_asm,exec,cpu_reset -D qemu.log build/runner/coparun-armv7thumb build/runner/test-armv7thumb.copapy build/runner/test.copapy-armv7thumb.bin + +qemu-arm -d in_asm,exec -D qemu_trace.log \ + -global driver=pl011.audiomaddr,property=addr,value=0xff7ec000 \ + -global driver=pl011.audiomaddr,property=size,value=0x100000 \ + your_binary +""" \ No newline at end of file From accb03f042c049ca78d07546f84c878ae2cdea9e Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 2 Mar 2026 21:31:07 +0100 Subject: [PATCH 15/27] Fix in test function "get_42" --- src/copapy/_math.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/copapy/_math.py b/src/copapy/_math.py index dd26bf6..fcd14c9 100644 --- a/src/copapy/_math.py +++ b/src/copapy/_math.py @@ -310,7 +310,7 @@ def get_42(x: value[Any]) -> value[float]: ... def get_42(x: NumLike) -> value[float] | float: """Returns the value representing the constant 42""" if isinstance(x, value): - return add_op('get_42', [x, x]) + return add_op('get_42', [x]) return float((int(x) * 3.0 + 42.0) * 5.0 + 21.0) From d2069d5d07b9b3268c9e12d6e92fc7f8c2d35619 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 2 Mar 2026 21:32:12 +0100 Subject: [PATCH 16/27] build script for local stencil builds updated for ARM64, ARM-Thumb and ARM-CortexM-Thumb --- tools/build.sh | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/tools/build.sh b/tools/build.sh index 4af186a..28679fd 100644 --- a/tools/build.sh +++ b/tools/build.sh @@ -4,10 +4,10 @@ set -eu ARCH=${1:-x86_64} case "$ARCH" in - (x86_64|arm-v6|arm-v7|arm-v7-thumb|arm-v7m-thumb|all) + (x86_64|arm64|arm-v6|arm-v7|arm-v7-thumb|arm-v7m-thumb|all) ;; (*) - echo "Usage: $0 [x86_64|arm-v6|arm-v7|arm-v6-thumb|arm-v7m-thumb|all]" + echo "Usage: $0 [x86_64|arm64|arm-v6|arm-v7|arm-v6-thumb|arm-v7m-thumb|all]" exit 1 ;; esac @@ -42,13 +42,44 @@ if [[ "$ARCH" == "x86_64" || "$ARCH" == "all" ]]; then -o build/runner/coparun fi +####################################### +# ARM 64 +####################################### +if [[ "$ARCH" == "arm64" || "$ARCH" == "all" ]]; then + echo "--------------arm64----------------" + + LIBGCC=$(aarch64-linux-gnu-gcc -print-libgcc-file-name) + + aarch64-linux-gnu-gcc -fno-pic -ffunction-sections \ + -c $SRC -O3 -o build/stencils/stencils.o + + aarch64-linux-gnu-ld -r \ + build/stencils/stencils.o \ + build/musl/musl_objects_arm64.o \ + $LIBGCC \ + -o $DEST/stencils_arm64_O3.o + + aarch64-linux-gnu-objdump -d -x \ + $DEST/stencils_arm64_O3.o \ + > build/stencils/stencils_arm64_O3.asm + + aarch64-linux-gnu-gcc \ + -Wall -Wextra -Wconversion -Wsign-conversion -static \ + -Wshadow -Wstrict-overflow -O3 \ + -DENABLE_LOGGING \ + src/coparun/runmem.c \ + src/coparun/coparun.c \ + src/coparun/mem_man.c \ + -o build/runner/coparun-arm64 +fi + ####################################### # ARM v6 ####################################### if [[ "$ARCH" == "arm-v6" || "$ARCH" == "all" ]]; then echo "--------------arm-v6 32 bit----------------" - LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) + LIBGCC=$(arm-none-eabi-gcc -march=armv6 -mfpu=vfp -mfloat-abi=hard -marm -print-libgcc-file-name) arm-none-eabi-gcc -fno-pic -ffunction-sections \ -march=armv6 -mfpu=vfp -mfloat-abi=hard -marm \ @@ -81,7 +112,7 @@ fi if [[ "$ARCH" == "arm-v7" || "$ARCH" == "all" ]]; then echo "--------------arm-v7 32 bit----------------" - LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) + LIBGCC=$(arm-none-eabi-gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -print-libgcc-file-name) arm-none-eabi-gcc -fno-pic -ffunction-sections \ -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm \ @@ -114,7 +145,7 @@ fi if [[ "$ARCH" == "arm-v7-thumb" || "$ARCH" == "all" ]]; then echo "--------------arm-v7a-thumb 32 bit----------------" - LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) + LIBGCC=$(arm-none-eabi-gcc -march=armv7 -mfpu=vfp3 -mthumb -print-libgcc-file-name) arm-none-eabi-gcc -fno-pic -ffunction-sections \ -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb \ @@ -122,7 +153,7 @@ if [[ "$ARCH" == "arm-v7-thumb" || "$ARCH" == "all" ]]; then arm-none-eabi-ld -r \ build/stencils/stencils.o \ - build/musl/musl_objects_armv7.o \ + build/musl/musl_objects_armv7thumb.o \ $LIBGCC \ -o $DEST/stencils_armv7thumb_O3.o @@ -147,7 +178,7 @@ fi if [[ "$ARCH" == "arm-v7m-thumb" || "$ARCH" == "all" ]]; then echo "--------------arm-v7m-thumb 32 bit----------------" - LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) + LIBGCC=$(arm-none-eabi-gcc -march=armv7e-m -mfpu=fpv4-sp-d16 -mfloat-abi=hard -mthumb -print-libgcc-file-name) arm-none-eabi-gcc -fno-pic -ffunction-sections \ -march=armv7e-m -mfpu=fpv4-sp-d16 -mfloat-abi=hard -mthumb \ From dd7fb12c64e4ecbbef6b39955ed02ca8e62e1a5b Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 2 Mar 2026 21:32:45 +0100 Subject: [PATCH 17/27] Helper bash script added for debugging ARM thumb stencils --- tools/test_thumb_stancils.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 tools/test_thumb_stancils.sh diff --git a/tools/test_thumb_stancils.sh b/tools/test_thumb_stancils.sh new file mode 100755 index 0000000..7fec81e --- /dev/null +++ b/tools/test_thumb_stancils.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +bash tools/build.sh arm-v7-thumb +python tests/test_ops_armv7thumb.py +qemu-arm -d in_asm -D qemu.log build/runner/coparun-armv7thumb build/runner/test-armv7thumb.copapy build/runner/test.copapy-armv7thumb.bin +arm-none-eabi-objdump -D -b binary -marm -M force-thumb --adjust-vma=0xff7ed000 build/runner/test.copapy-armv7thumb.bin > build/runner/test.copapy-armv7thumb.asm \ No newline at end of file From 7f963d7e43d0ca555e01aee6c79bcc2d5be30ec9 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Mon, 2 Mar 2026 21:33:24 +0100 Subject: [PATCH 18/27] CI: Testing for ARMv7 extended to armv7thumb and armv7mthumb --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 349f366..9f6864e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -214,6 +214,10 @@ jobs: gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \ src/coparun/coparun.c src/coparun/mem_man.c && \ pytest && \ + export CP_TARGET_ARCH=armv7thumb && \ + pytest && \ + export CP_TARGET_ARCH=armv7mthumb && \ + pytest && \ bash tools/create_asm.sh" - uses: actions/upload-artifact@v4 From 7a3088ec48a8ddf95792d0ec66f9c64c06b291fc Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 3 Mar 2026 13:05:47 +0100 Subject: [PATCH 19/27] added add_sign_int32 function, since pelfy returns addend-values for 32 bit x86 as unsigned int32 --- src/copapy/_stencils.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/copapy/_stencils.py b/src/copapy/_stencils.py index f9bd866..001d15d 100644 --- a/src/copapy/_stencils.py +++ b/src/copapy/_stencils.py @@ -135,6 +135,11 @@ def get_op_after_last_call_in_function(func: pelfy.elf_symbol) -> int: return reloc.fields['r_offset'] - func.offset_in_section + 4 +def add_sign_int32(value: int) -> int: + """Convert a 32-bit unsigned integer to a signed integer.""" + return value - 0x100000000 if value > 0x7FFFFFFF else value + + class stencil_database(): """A class for loading and querying a stencil database from an ELF object file @@ -226,6 +231,7 @@ class stencil_database(): cache.append(reloc_entry) yield reloc_entry + def get_patch(self, relocation: relocation_entry, symbol_address: int, function_offset: int, symbol_type: int) -> patch_entry: """Return patch positions for a provided symbol (function or object) @@ -251,12 +257,14 @@ class stencil_database(): if pr.type.endswith('64_PC32') or pr.type.endswith('64_PLT32'): # S + A - P - patch_value = symbol_address + pr.fields['r_addend'] - patch_offset + addend = add_sign_int32(pr.fields['r_addend']) + patch_value = symbol_address + addend - patch_offset #print(f" *> {pr.type} {patch_value=} {symbol_address=} {pr.fields['r_addend']=} {pr.bits=}, {function_offset=} {patch_offset=}") elif pr.type == 'R_386_PC32': # S + A - P - patch_value = symbol_address + pr.fields['r_addend'] - patch_offset + addend = add_sign_int32(pr.fields['r_addend']) + patch_value = symbol_address + addend - patch_offset #print(f" *> {pr.type} {pr.symbol.name} {patch_value=} {symbol_address=} {pr.fields['r_addend']=} {bin(pr.fields['r_addend'])} {pr.bits=}, {function_offset=} {patch_offset=}") elif pr.type == 'R_386_32': From 04cdf50a04f7ec62a7d08b2de93faa217230307b Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 3 Mar 2026 13:24:46 +0100 Subject: [PATCH 20/27] Updated pelfy dependency --- .github/workflows/ci.yml | 8 ++++---- pyproject.toml | 18 +++++++++++------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f6864e..e1c3203 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,7 +151,7 @@ jobs: - name: Use ARM64 container run: | docker run --rm -v $PWD:/app -w /app --platform linux/arm64 ghcr.io/nonannet/arm64_test:1 \ - bash -lc "pip install . && \ + bash -lc "pip install .[mindev] && \ mkdir -p build/runner && \ gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \ src/coparun/coparun.c src/coparun/mem_man.c && \ @@ -180,7 +180,7 @@ jobs: - name: Use ARMv6 container run: | docker run --rm -v $PWD:/app -w /app --platform linux/arm/v6 ghcr.io/nonannet/armv6_test:1 \ - bash -lc "pip install . && \ + bash -lc "pip install .[mindev] && \ mkdir -p build/runner && \ gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \ src/coparun/coparun.c src/coparun/mem_man.c && \ @@ -209,7 +209,7 @@ jobs: - name: Use ARMv7 container run: | docker run --rm -v $PWD:/app -w /app --platform linux/arm/v7 ghcr.io/nonannet/armv7_test:1 \ - bash -lc "pip install . && \ + bash -lc "pip install .[mindev] && \ mkdir -p build/runner && \ gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \ src/coparun/coparun.c src/coparun/mem_man.c && \ @@ -248,7 +248,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Python dependencies - run: python -m pip install .[dev] + run: python -m pip install .[mindev] - name: Set up MSVC environment uses: microsoft/setup-msbuild@v2 diff --git a/pyproject.toml b/pyproject.toml index d5643c8..1ee53ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "copapy" dynamic = ["version"] authors = [ - { name="Nicolas Kruse", email="nicolas.kruse@nonan.net" }, + { name="Nicolas Kruse", email="nicolas.kruse@nonan.net" }, ] description = "Copy-Patch Compiler" readme = "README.md" @@ -45,14 +45,18 @@ dev = [ "ruff", "mypy", "pytest", - "pelfy>=1.0.7" + "pelfy>=1.0.8" +] +mindev = [ + "pytest", + "pelfy>=1.0.8" ] doc_build = [ - "sphinx", - "pydata_sphinx_theme", - "sphinx-autodoc-typehints", - "myst-parser", - "pelfy>=1.0.7" + "sphinx", + "pydata_sphinx_theme", + "sphinx-autodoc-typehints", + "myst-parser", + "pelfy>=1.0.8" ] [tool.mypy] From c31601853b19ffd0f997081227e5564806fdfcd8 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 3 Mar 2026 13:31:22 +0100 Subject: [PATCH 21/27] CI: LIBGCC version fixed for thumb builds --- tools/crosscompile.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/crosscompile.sh b/tools/crosscompile.sh index 29a15ca..78a4f2d 100644 --- a/tools/crosscompile.sh +++ b/tools/crosscompile.sh @@ -43,12 +43,12 @@ arm-none-eabi-ld -r $STMP /object_files/musl_objects_armv7.o $LIBGCC -o $DEST/st # ARMv7 Thumb for Cortex-A with hardware fp arm-none-eabi-gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb $FLAGS -$OPT -c $SRC -o $STMP -LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) +LIBGCC=$(arm-none-eabi-gcc -march=armv7 -mfpu=vfp3 -mthumb -print-libgcc-file-name) arm-none-eabi-ld -r $STMP /object_files/musl_objects_armv7thumb.o $LIBGCC -o $DEST/stencils_armv7thumb_$OPT.o # Armv7 Thumb for Cortex-M3..7 hardware fp arm-none-eabi-gcc -march=armv7e-m -mfpu=fpv4-sp-d16 -mfloat-abi=hard -mthumb $FLAGS -$OPT -c $SRC -o $STMP -LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) +LIBGCC=$(arm-none-eabi-gcc -march=armv7e-m -mfpu=fpv4-sp-d16 -mfloat-abi=hard -mthumb -print-libgcc-file-name) arm-none-eabi-ld -r $STMP /object_files/musl_objects_armv7mthumb.o $LIBGCC -o $DEST/stencils_armv7mthumb_$OPT.o # PowerPC64LE From 0212fa77a3d5f1b94db27d35fedd15aa601a7a25 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 3 Mar 2026 14:25:50 +0100 Subject: [PATCH 22/27] type annotation for data_writer.copy() fixed --- src/copapy/_binwrite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/copapy/_binwrite.py b/src/copapy/_binwrite.py index 1af2ee3..db1f15f 100644 --- a/src/copapy/_binwrite.py +++ b/src/copapy/_binwrite.py @@ -25,7 +25,7 @@ class data_writer(): self._data: list[tuple[str, bytes, int]] = [] self.byteorder: ByteOrder = byteorder - def copy(self): + def copy(self) -> 'data_writer': cp = data_writer(self.byteorder) cp._data = self._data.copy() return cp From b7d5f3a1297bd9812138066d64d18f6224054670 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 3 Mar 2026 14:47:53 +0100 Subject: [PATCH 23/27] exclude non-stencil functions in the test "test_start_end_function" --- tests/test_stencil_db.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_stencil_db.py b/tests/test_stencil_db.py index 0ef40e4..7e762bd 100644 --- a/tests/test_stencil_db.py +++ b/tests/test_stencil_db.py @@ -19,11 +19,18 @@ def test_start_end_function(): if symbol.relocations and symbol.relocations[-1].symbol.info == 'STT_NOTYPE': - print('-', sym_name, get_stencil_position(symbol), len(symbol.data)) + if symbol.section and symbol.section.name == '.text': + print('SKIP', sym_name, '(Aux function, not a stencil)') + continue - start, end = get_stencil_position(symbol) + if symbol.section: + function_size = symbol.section.fields['sh_size'] # len(symbol.data) excludes nop after the function - assert start >= 0 and end >= start and end <= len(symbol.data) + print('-', sym_name, get_stencil_position(symbol), function_size) + + start, end = get_stencil_position(symbol) + + assert (start >= 0 and end >= start and end <= function_size) def test_aux_functions(): From c6fd69d61b112878d57b87bfb4022f0300fc5b65 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 3 Mar 2026 15:34:37 +0100 Subject: [PATCH 24/27] CI: different armv7 variates separated --- .github/workflows/ci.yml | 80 +++++++++++++++++++++++++++++++++++++--- tools/build.sh | 7 +++- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1c3203..1cd9751 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -211,13 +211,13 @@ jobs: docker run --rm -v $PWD:/app -w /app --platform linux/arm/v7 ghcr.io/nonannet/armv7_test:1 \ bash -lc "pip install .[mindev] && \ mkdir -p build/runner && \ - gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \ + gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \ + -Wall -Wextra -Wconversion -Wsign-conversion \ + -Wshadow -Wstrict-overflow -O3 \ + -DENABLE_LOGGING \ + -o build/runner/coparun src/coparun/runmem.c \ src/coparun/coparun.c src/coparun/mem_man.c && \ pytest && \ - export CP_TARGET_ARCH=armv7thumb && \ - pytest && \ - export CP_TARGET_ARCH=armv7mthumb && \ - pytest && \ bash tools/create_asm.sh" - uses: actions/upload-artifact@v4 @@ -225,6 +225,74 @@ jobs: name: runner-linux-armv7 path: build/runner/* + build-armv7thumb: + needs: [build_stencils] + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: stencil-object-files + path: src/copapy/obj + - name: Set up QEMU for ARMv7 + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/arm/v7 + - name: Use ARMv7 container + run: | + docker run --rm -v $PWD:/app -w /app --platform linux/arm/v7 ghcr.io/nonannet/armv7_test:1 \ + bash -lc "pip install .[mindev] && \ + mkdir -p build/runner && \ + gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \ + -Wall -Wextra -Wconversion -Wsign-conversion \ + -Wshadow -Wstrict-overflow -O3 \ + -DENABLE_LOGGING \ + -o build/runner/coparun src/coparun/runmem.c \ + src/coparun/coparun.c src/coparun/mem_man.c && \ + export CP_TARGET_ARCH=armv7thumb && \ + pytest && \ + bash tools/create_asm.sh" + + - uses: actions/upload-artifact@v4 + with: + name: runner-linux-armv7thumb + path: build/runner/* + + build-armv7mthumb: + needs: [build_stencils] + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: stencil-object-files + path: src/copapy/obj + - name: Set up QEMU for ARMv7 + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/arm/v7 + - name: Use ARMv7 container + run: | + docker run --rm -v $PWD:/app -w /app --platform linux/arm/v7 ghcr.io/nonannet/armv7_test:1 \ + bash -lc "pip install .[mindev] && \ + mkdir -p build/runner && \ + gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \ + -Wall -Wextra -Wconversion -Wsign-conversion \ + -Wshadow -Wstrict-overflow -O3 \ + -DENABLE_LOGGING \ + -o build/runner/coparun src/coparun/runmem.c \ + src/coparun/coparun.c src/coparun/mem_man.c && \ + export CP_TARGET_ARCH=armv7mthumb && \ + pytest && \ + bash tools/create_asm.sh" + + - uses: actions/upload-artifact@v4 + with: + name: runner-linux-armv7mthumb + path: build/runner/* + build-windows: needs: [build_stencils] runs-on: windows-latest @@ -307,6 +375,8 @@ jobs: cp tmp/runner-linux-arm64/coparun release/coparun-aarch64 cp tmp/runner-linux-armv6/coparun release/coparun-armv6 cp tmp/runner-linux-armv7/coparun release/coparun-armv7 + cp tmp/runner-linux-armv7thumb/coparun release/coparun-armv7thumb + cp tmp/runner-linux-armv7mthumb/coparun release/coparun-armv7mthumb cp tmp/runner-win/coparun*.exe release/ TAG="${{ steps.version.outputs.version }}" diff --git a/tools/build.sh b/tools/build.sh index 28679fd..4686e3d 100644 --- a/tools/build.sh +++ b/tools/build.sh @@ -128,6 +128,7 @@ if [[ "$ARCH" == "arm-v7" || "$ARCH" == "all" ]]; then $DEST/stencils_armv7_O3.o \ > build/stencils/stencils_armv7_O3.asm + # The same runner for all ARM7 arm-linux-gnueabihf-gcc \ -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \ -Wall -Wextra -Wconversion -Wsign-conversion \ @@ -161,8 +162,9 @@ if [[ "$ARCH" == "arm-v7-thumb" || "$ARCH" == "all" ]]; then $DEST/stencils_armv7thumb_O3.o \ > build/stencils/stencils_armv7thumb_O3.asm + # The same runner for all ARM7 arm-linux-gnueabihf-gcc \ - -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb -static \ + -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -static \ -Wall -Wextra -Wconversion -Wsign-conversion \ -Wshadow -Wstrict-overflow -O3 \ -DENABLE_LOGGING \ @@ -194,8 +196,9 @@ if [[ "$ARCH" == "arm-v7m-thumb" || "$ARCH" == "all" ]]; then $DEST/stencils_armv7mthumb_O3.o \ > build/stencils/stencils_armv7mthumb_O3.asm + # The same runner for all ARM7 arm-linux-gnueabihf-gcc \ - -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb -static \ + -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -static \ -Wall -Wextra -Wconversion -Wsign-conversion \ -Wshadow -Wstrict-overflow -O3 \ -DENABLE_LOGGING \ From e48fc7c485d416b2566fcb10d1badf02fcd11b83 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 3 Mar 2026 16:09:06 +0100 Subject: [PATCH 25/27] test for arm thumb variants split into two tests --- tests/test_ops_armv7mthumb.py | 174 ++++++++++++++++++++++++++++++++++ tests/test_ops_armv7thumb.py | 10 +- 2 files changed, 179 insertions(+), 5 deletions(-) create mode 100644 tests/test_ops_armv7mthumb.py diff --git a/tests/test_ops_armv7mthumb.py b/tests/test_ops_armv7mthumb.py new file mode 100644 index 0000000..16058d6 --- /dev/null +++ b/tests/test_ops_armv7mthumb.py @@ -0,0 +1,174 @@ +from copapy import NumLike, iif, value +from copapy.backend import Store, compile_to_dag, add_read_value_remote +import subprocess +from copapy import _binwrite +import copapy.backend as backend +import os +import warnings +import re +import struct +import pytest +import copapy as cp + +if os.name == 'nt': + # On Windows wsl and qemu-user is required: + # sudo apt install qemu-user + qemu_command = ['wsl', 'qemu-arm'] +else: + qemu_command = ['qemu-arm'] + + +def parse_results(log_text: str) -> dict[int, bytes]: + regex = r"^READ_DATA offs=(\d*) size=(\d*) data=(.*)$" + matches = re.finditer(regex, log_text, re.MULTILINE) + var_dict: dict[int, bytes] = {} + + for match in matches: + value_str: list[str] = match.group(3).strip().split(' ') + #print('--', value_str) + value = bytes(int(v, base=16) for v in value_str) + if len(value) <= 8: + var_dict[int(match.group(1))] = value + + return var_dict + + +def run_command(command: list[str]) -> str: + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8', check=False) + assert result.returncode != 11, f"SIGSEGV (segmentation fault)\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + assert result.returncode == 0, f"\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + return result.stdout + + +def check_for_qemu() -> bool: + command = qemu_command + ['--version'] + try: + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + except Exception: + return False + return result.returncode == 0 + + +def function1(c1: NumLike) -> list[NumLike]: + return [c1 / 4, c1 / -4, c1 // 4, c1 // -4, (c1 * -1) // 4, + c1 * 4, c1 * -4, + c1 + 4, c1 - 4, + c1 > 2, c1 > 100, c1 < 4, c1 < 100] + +def function1ex(c1: NumLike) -> list[NumLike]: + return [c1 // 4] + + +def function2(c1: NumLike) -> list[NumLike]: + return [c1 * 4.44, c1 * -4.44] + + +def function3(c1: NumLike) -> list[NumLike]: + return [c1 / 4] + + +def function4(c1: NumLike) -> list[NumLike]: + return [c1 == 9, c1 == 4, c1 != 9, c1 != 4] + + +def function5(c1: NumLike) -> list[NumLike]: + return [c1 == True, c1 == False, c1 != True, c1 != False, c1 / 2, c1 + 2] + + +def function6(c1: NumLike) -> list[NumLike]: + return [c1 == True] + + +def iiftests(c1: NumLike) -> list[NumLike]: + return [iif(c1 > 5, 8, 9), + iif(c1 < 5, 8.5, 9.5), + iif(1 > 5, 3.3, 8.8) + c1, + iif(1 < 5, c1 * 3.3, 8.8), + iif(c1 < 5, c1 * 3.3, 8.8)] + + +@pytest.mark.runner +def test_compile(): + c_i = value(9) + c_f = value(1.111) + c_b = value(True) + + ret_test = function1(c_i) + function1(c_f) + function2(c_i) + function2(c_f) + function3(c_i) + function4(c_i) + function5(c_b) + [value(9) % 2] + iiftests(c_i) + iiftests(c_f) + [cp.asin(c_i/10)] + ret_ref = function1(9) + function1(1.111) + function2(9) + function2(1.111) + function3(9) + function4(9) + function5(True) + [9 % 2] + iiftests(9) + iiftests(1.111) + [cp.asin(9/10)] + + out = [Store(r) for r in ret_test] + + sdb = backend.stencil_db_from_package('armv7mthumb') + dw, variables = compile_to_dag(out, sdb) + + #dw.write_com(_binwrite.Command.READ_DATA) + #dw.write_int(0) + #dw.write_int(28) + + du = dw.copy() + dw.write_com(_binwrite.Command.RUN_PROG) + du.write_com(_binwrite.Command.DUMP_CODE) + + for v in ret_test: + assert isinstance(v, value) + add_read_value_remote(dw, variables, v.net) + + #dw.write_com(_binwrite.Command.READ_DATA) + #dw.write_int(0) + #dw.write_int(28) + + dw.write_com(_binwrite.Command.END_COM) + du.write_com(_binwrite.Command.END_COM) + + #print('* Data to runner:') + #dw.print() + + dw.to_file('build/runner/test-armv7mthumb.copapy') + du.to_file('build/runner/test-armv7mthumb-dump.copapy') + + if not check_for_qemu(): + warnings.warn("qemu-armv7 not found, armv7 test skipped!", UserWarning) + return + if not os.path.isfile('build/runner/coparun-armv7'): + warnings.warn("armv7 runner not found, armv7 test skipped!", UserWarning) + return + + print('----- Dump code...') + command = qemu_command + ['build/runner/coparun-armv7', 'build/runner/test-armv7mthumb-dump.copapy', 'build/runner/test.copapy-armv7mthumb.bin'] + result = run_command(command) + + print('----- Run code...') + command = qemu_command + ['build/runner/coparun-armv7', 'build/runner/test-armv7mthumb.copapy'] + result = run_command(command) + + + print('* Output from runner:\n--') + print(result) + print('--') + + assert 'Return value: 1' in result + + result_data = parse_results(result) + + for test, ref in zip(ret_test, ret_ref): + assert isinstance(test, value) + address = variables[test.net][0] + data = result_data[address] + if test.dtype == 'int': + val = int.from_bytes(data, sdb.byteorder, signed=True) + elif test.dtype == 'bool': + val = bool.from_bytes(data, sdb.byteorder) + elif test.dtype == 'float': + en = {'little': '<', 'big': '>'}[sdb.byteorder] + val = struct.unpack(en + 'f', data)[0] + assert isinstance(val, float) + else: + raise Exception(f"Unknown type: {test.dtype}") + print('+', val, ref, test.dtype, f" addr={address}") + for t in (int, float, bool): + assert isinstance(val, t) == isinstance(ref, t), f"Result type does not match for {val} and {ref}" + assert val == pytest.approx(ref, 1e-5), f"Result does not match: {val} and reference: {ref}" # pyright: ignore[reportUnknownMemberType] + + +if __name__ == "__main__": + test_compile() diff --git a/tests/test_ops_armv7thumb.py b/tests/test_ops_armv7thumb.py index 983510e..997f1cd 100644 --- a/tests/test_ops_armv7thumb.py +++ b/tests/test_ops_armv7thumb.py @@ -98,7 +98,7 @@ def test_compile(): out = [Store(r) for r in ret_test] - sdb = backend.stencil_db_from_package('armv7mthumb') + sdb = backend.stencil_db_from_package('armv7thumb') dw, variables = compile_to_dag(out, sdb) #dw.write_com(_binwrite.Command.READ_DATA) @@ -129,16 +129,16 @@ def test_compile(): if not check_for_qemu(): warnings.warn("qemu-armv7 not found, armv7 test skipped!", UserWarning) return - if not os.path.isfile('build/runner/coparun-armv7thumb'): - warnings.warn("armv7thumb runner not found, armv7thumb test skipped!", UserWarning) + if not os.path.isfile('build/runner/coparun-armv7'): + warnings.warn("armv7 runner not found, armv7 test skipped!", UserWarning) return print('----- Dump code...') - command = qemu_command + ['build/runner/coparun-armv7thumb', 'build/runner/test-armv7thumb-dump.copapy', 'build/runner/test.copapy-armv7thumb.bin'] + command = qemu_command + ['build/runner/coparun-armv7', 'build/runner/test-armv7thumb-dump.copapy', 'build/runner/test.copapy-armv7thumb.bin'] result = run_command(command) print('----- Run code...') - command = qemu_command + ['build/runner/coparun-armv7thumb', 'build/runner/test-armv7thumb.copapy'] + command = qemu_command + ['build/runner/coparun-armv7', 'build/runner/test-armv7thumb.copapy'] result = run_command(command) From 031249241e2894c43d7bb10bd04f0397c28b5438 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Tue, 3 Mar 2026 16:56:59 +0100 Subject: [PATCH 26/27] Removed un-required bulk stack allocation and added 4 Byte alignment check. CI updated. --- .github/workflows/ci.yml | 18 ++++++++++-------- src/copapy/_stencils.py | 3 +++ stencils/generate_stencils.py | 4 ++-- tools/create_asm.sh | 7 +++++-- tools/test_example_code.sh | 21 +++++++++++++++++++++ 5 files changed, 41 insertions(+), 12 deletions(-) create mode 100755 tools/test_example_code.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cd9751..44eabea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,7 +180,8 @@ jobs: - name: Use ARMv6 container run: | docker run --rm -v $PWD:/app -w /app --platform linux/arm/v6 ghcr.io/nonannet/armv6_test:1 \ - bash -lc "pip install .[mindev] && \ + bash -lc "set -x && \ + pip install .[mindev] && \ mkdir -p build/runner && \ gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \ src/coparun/coparun.c src/coparun/mem_man.c && \ @@ -209,9 +210,10 @@ jobs: - name: Use ARMv7 container run: | docker run --rm -v $PWD:/app -w /app --platform linux/arm/v7 ghcr.io/nonannet/armv7_test:1 \ - bash -lc "pip install .[mindev] && \ + bash -lc "set -x && \ + pip install .[mindev] && \ mkdir -p build/runner && \ - gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \ + gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \ -Wall -Wextra -Wconversion -Wsign-conversion \ -Wshadow -Wstrict-overflow -O3 \ -DENABLE_LOGGING \ @@ -242,7 +244,8 @@ jobs: - name: Use ARMv7 container run: | docker run --rm -v $PWD:/app -w /app --platform linux/arm/v7 ghcr.io/nonannet/armv7_test:1 \ - bash -lc "pip install .[mindev] && \ + bash -lc "set -x && \ + pip install .[mindev] && \ mkdir -p build/runner && \ gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \ -Wall -Wextra -Wconversion -Wsign-conversion \ @@ -276,7 +279,8 @@ jobs: - name: Use ARMv7 container run: | docker run --rm -v $PWD:/app -w /app --platform linux/arm/v7 ghcr.io/nonannet/armv7_test:1 \ - bash -lc "pip install .[mindev] && \ + bash -lc "set -x && \ + pip install .[mindev] && \ mkdir -p build/runner && \ gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \ -Wall -Wextra -Wconversion -Wsign-conversion \ @@ -342,7 +346,7 @@ jobs: path: build/runner/* release-stencils: - needs: [build_stencils, build-ubuntu, build-windows, build-arm64, build-armv6, build-armv7] + needs: [build_stencils, build-ubuntu, build-windows, build-arm64, build-armv6, build-armv7, build-armv7thumb, build-armv7mthumb] runs-on: ubuntu-latest if: github.ref == 'refs/heads/main' && github.event_name == 'push' permissions: @@ -375,8 +379,6 @@ jobs: cp tmp/runner-linux-arm64/coparun release/coparun-aarch64 cp tmp/runner-linux-armv6/coparun release/coparun-armv6 cp tmp/runner-linux-armv7/coparun release/coparun-armv7 - cp tmp/runner-linux-armv7thumb/coparun release/coparun-armv7thumb - cp tmp/runner-linux-armv7mthumb/coparun release/coparun-armv7mthumb cp tmp/runner-win/coparun*.exe release/ TAG="${{ steps.version.outputs.version }}" diff --git a/src/copapy/_stencils.py b/src/copapy/_stencils.py index 001d15d..7ac84e5 100644 --- a/src/copapy/_stencils.py +++ b/src/copapy/_stencils.py @@ -343,6 +343,9 @@ class stencil_database(): elif pr.type.endswith('_ABS32'): # R_ARM_ABS32 # S + A (replaces full 32 bit) + assert not patch_offset % 4, 'R_ARM_ABS32 patched data like literals needs to be 4 Byte aligned' + # This might be caused by the call in entry_function_shell if not aligned + patch_value = symbol_address + pr.fields['r_addend'] symbol_type = symbol_type + 0x03 # Relative to data section diff --git a/stencils/generate_stencils.py b/stencils/generate_stencils.py index 19b0f73..307db55 100644 --- a/stencils/generate_stencils.py +++ b/stencils/generate_stencils.py @@ -57,8 +57,8 @@ def norm_indent(f: Callable[..., str]) -> Callable[..., str]: def get_entry_function_shell() -> str: return f""" {entry_func_prefix}int entry_function_shell(){{ - volatile char stack_place_holder[{stack_size}]; - stack_place_holder[0] = 0; + //volatile char stack_place_holder[{stack_size}]; + //stack_place_holder[0] = 0; result_int(0); return 1; }} diff --git a/tools/create_asm.sh b/tools/create_asm.sh index ca6e220..202cd66 100644 --- a/tools/create_asm.sh +++ b/tools/create_asm.sh @@ -10,7 +10,6 @@ cparch=$(python3 -c "import copapy; print(copapy._stencils.detect_process_arch() # Disassemble stencil object file objdump -d -x src/copapy/obj/stencils_${cparch}_O3.o > build/runner/stencils.asm -# Create example code disassembly python3 tools/make_example.py build/runner/coparun build/runner/test.copapy build/runner/test.copapy.bin @@ -28,6 +27,10 @@ fi echo "Archtitecture: '$cparch'" -objdump -D -b binary -m $cparch --adjust-vma=0x10000 build/runner/test.copapy.bin > build/runner/example.asm +if [[ "$cparch" == *"thumb"* ]]; then + objdump -D -b binary -marm -M force-thumb --adjust-vma=0x10000 build/runner/test.copapy.bin > build/runner/example.asm +else + objdump -D -b binary -m $cparch --adjust-vma=0x10000 build/runner/test.copapy.bin > build/runner/example.asm +fi rm build/runner/test.copapy.bin diff --git a/tools/test_example_code.sh b/tools/test_example_code.sh new file mode 100755 index 0000000..7b0c054 --- /dev/null +++ b/tools/test_example_code.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Build arm-v7 runner and stencils +bash tools/build.sh arm-v7 + +# Build arm-v7-thumb stencils +bash tools/build.sh arm-v7-thumb + +# Build arm-v7-thumb example code +export CP_TARGET_ARCH=armv7thumb +python3 tools/make_example.py +build/runner/coparun-armv7 build/runner/test.copapy build/runner/test.copapy.bin + +arm-none-eabi-objdump -D -b binary -marm -M force-thumb --adjust-vma=0x1000000 build/runner/test.copapy.bin > build/runner/test.copapy-example-armv7thumb.asm + +# Build arm-v7-thumb example code +export CP_TARGET_ARCH=armv7 +python3 tools/make_example.py +build/runner/coparun-armv7 build/runner/test.copapy build/runner/test.copapy.bin + +arm-none-eabi-objdump -D -b binary -marm --adjust-vma=0x1000000 build/runner/test.copapy.bin > build/runner/test.copapy-example-armv7.asm \ No newline at end of file From 01f02cc9ba21e0057cc58d8af7e4af5508f88724 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 4 Mar 2026 15:16:51 +0100 Subject: [PATCH 27/27] Readme updated --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0df2f75..d7780ca 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ The main features can be summarized as: - Memory and type safety with a minimal set of runtime errors - Deterministic execution - Automatic differentiation for efficient realtime optimization (reverse-mode) -- Optimized machine code for x86_64, ARMv6, ARMv7 and AArch64 +- Optimized machine code for x86_64, 32 Bit ARM (Cortex-A and Cortex-M) and AArch64 - Highly portable to new architectures - Small Python package with minimal dependencies and no cross-compile toolchain required @@ -31,7 +31,6 @@ While hardware I/O is obviously a core aspect of the project, it is not yet avai Currently in development: - Array stencils for handling very large arrays and generating SIMD-optimized code - e.g., for machine vision and neural network applications -- Support for Thumb instructions required by ARM*-M targets (for MCUs) - Constant regrouping for further symbolic optimization of the computation graph Despite missing SIMD-optimization, benchmark performance shows promising numbers. The following chart plots the results in comparison to NumPy 2.3.5: @@ -253,4 +252,4 @@ This project is licensed under the MIT license - see the [LICENSE](LICENSE) file [^2]: The compiler must support tail-call optimization (TCO). Currently, GCC is supported. Porting to a new architecture requires implementing a subset of relocation types used by that architecture. -[^3]: Supported architectures: x86_64, AArch64, ARMv6 and 7 (non-Thumb). ARMv6/7-M (Thumb) support is in development. Code for x86 32-bit exists but has unresolved issues and a low priority. +[^3]: Supported architectures: x86_64, AArch64, ARMv6/7 (non-Thumb) and ARMv7 Thumb for Cortex-A and Cortex-M. Code for x86 32-bit exists but has unresolved issues and a low priority.