From a9b52bcf2481dac24cf00f52467a769347f16588 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Thu, 30 Oct 2025 10:55:23 +0100 Subject: [PATCH] Cross compilation for aarch64-runner added --- .github/workflows/build_wheels.yml | 2 + .github/workflows/ci.yml | 11 ++++ src/copapy/_stencils.py | 5 +- src/copapy/backend.py | 3 +- tests/test_compile_aarch64.py | 88 ++++++++++++++++++++++++++++++ tools/crosscompile.sh | 8 +++ tools/extract_code.py | 25 ++++++--- tools/get_binaries.py | 2 + tools/make_example.py | 40 +++++++++++++- 9 files changed, 172 insertions(+), 12 deletions(-) create mode 100644 tests/test_compile_aarch64.py diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 5240e7b..cafb86a 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -23,6 +23,7 @@ jobs: path: src/copapy/obj/*.o build_wheels: + if: contains(github.ref, '-beta') == false needs: [build_stencils] runs-on: ${{ matrix.os }} strategy: @@ -65,6 +66,7 @@ jobs: path: wheelhouse/*.whl # publish: +# if: contains(github.ref, '-beta') == false # needs: [build_wheels] # runs-on: ubuntu-latest # steps: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64101f0..48d86d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,11 @@ jobs: name: stencil-object-files path: src/copapy/obj/*.o + - uses: actions/upload-artifact@v4 + with: + name: cross-runner + path: bin/coparun-* + build-ubuntu: needs: [build_stencils] runs-on: ubuntu-latest @@ -71,6 +76,11 @@ jobs: echo '

test.copapy.asm

' >> $GITHUB_STEP_SUMMARY python tools/clean_asm.py bin/test.copapy.asm >> $GITHUB_STEP_SUMMARY + python tools/extract_code.py "bin/test-aarch64.copapy" "bin/test-aarch64.copapy.bin" + aarch64-linux-gnu-objdump -D -b binary -m aarch64 --adjust-vma=0x1000 bin/test-aarch64.copapy.bin > bin/test-aarch64.copapy.asm + echo '

test-aarch64.copapy.asm

' >> $GITHUB_STEP_SUMMARY + python tools/clean_asm.py bin/test-aarch64.copapy.asm >> $GITHUB_STEP_SUMMARY + objdump -d -x src/copapy/obj/stencils_x86_64_O3.o > bin/stencils_x86_64_O3.asm echo '

stencils_x86_64_O3.asm

' >> $GITHUB_STEP_SUMMARY python tools/clean_asm.py bin/stencils_x86_64_O3.asm >> $GITHUB_STEP_SUMMARY @@ -176,6 +186,7 @@ jobs: set -v mkdir -p release cp tmp/stencil-object-files/*.o release/ + cp tmp/cross-runner/coparun-* release/ cp tmp/runner-linux/coparun release/ cp tmp/runner-win/coparun.exe release/ diff --git a/src/copapy/_stencils.py b/src/copapy/_stencils.py index 42d3100..869f2a3 100644 --- a/src/copapy/_stencils.py +++ b/src/copapy/_stencils.py @@ -27,12 +27,13 @@ class patch_entry: def translate_relocation(reloc: pelfy.elf_relocation, offset: int) -> patch_entry: - if reloc.type in ('R_AMD64_PLT32', 'R_AMD64_PC32'): + if reloc.type.endswith('_PLT32') or reloc.type.endswith('_PC32'): # S + A - P mask = 0xFFFFFFFF # 32 bit imm = offset - elif reloc.type.endswith('_JUMP26'): + elif reloc.type.endswith('_JUMP26') or reloc.type.endswith('_CALL26'): + # S + A - P assert reloc.file.byteorder == 'little', "Big endian not supported for ARM64" mask = 0x3ffffff # 26 bit imm = offset >> 2 diff --git a/src/copapy/backend.py b/src/copapy/backend.py index 6b02056..839bef9 100644 --- a/src/copapy/backend.py +++ b/src/copapy/backend.py @@ -1,5 +1,5 @@ from ._target import add_read_command -from ._basic_types import Net, Op, Node, CPConstant, Write +from ._basic_types import Net, Op, Node, CPConstant, Write, stencil_db_from_package from ._compiler import compile_to_dag, \ stable_toposort, get_const_nets, get_all_dag_edges, add_read_ops, \ add_write_ops @@ -17,4 +17,5 @@ __all__ = [ "get_all_dag_edges", "add_read_ops", "add_write_ops", + "stencil_db_from_package" ] diff --git a/tests/test_compile_aarch64.py b/tests/test_compile_aarch64.py new file mode 100644 index 0000000..09c9097 --- /dev/null +++ b/tests/test_compile_aarch64.py @@ -0,0 +1,88 @@ +from copapy import variable, NumLike +from copapy.backend import Write, compile_to_dag, add_read_command +import subprocess +import struct +from copapy import _binwrite +import copapy.backend as backend +import os +import pytest + +if os.name == 'nt': + # On Windows wsl and qemu-user is required: + # sudo apt install qemu-user + qemu_command = ['wsl', 'qemu-aarch64', 'bin/coparun-aarch64', 'bin/test.copapy'] +else: + qemu_command = ['qemu-aarch64', 'bin/coparun-aarch64', 'bin/test.copapy'] + + +def run_command(command: list[str]) -> str: + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8', check=False) + assert result.returncode != 11, f"SIGSEGV (segmentation fault)\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + assert result.returncode == 0, f"\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + return result.stdout + + +def test_example(): + c1 = 4 + c2 = 2 + + i1 = c1 * 2 + r1 = i1 + 7 + (c2 + 7 * 9) + r2 = i1 + 9 + + en = {'little': '<', 'big': '>'}['little'] + data = struct.pack(en + 'i', r1) + print("example r1 " + ' '.join(f'{b:02X}' for b in data)) + + data = struct.pack(en + 'i', r2) + print("example r2 " + ' '.join(f'{b:02X}' for b in data)) + + +def function(c1: NumLike, c2: NumLike) -> tuple[NumLike, ...]: + i1 = c1 // 3.3 + 5 + i2 = c2 * 5 + c1 + r1 = i1 + i2 * 55 / 4 + r2 = 4 * i2 + 5 + + return i1, i2, r1, r2 + + +@pytest.mark.skip(reason="no way of currently testing this") +def test_compile(): + c1 = variable(4) + c2 = variable(2) + + ret = function(c1, c2) + #ret = [c1 // 3.3 + 5] + + out = [Write(r) for r in ret] + + sdb = backend.stencil_db_from_package('aarch64', 'O3') + il, variables = compile_to_dag(out, sdb) + + # run program command + il.write_com(_binwrite.Command.RUN_PROG) + + for net in ret: + assert isinstance(net, backend.Net) + add_read_command(il, variables, net) + + il.write_com(_binwrite.Command.END_COM) + + print('* Data to runner:') + il.print() + + il.to_file('bin/test.copapy') + + result = run_command(qemu_command) + print('* Output from runner:\n--') + print(result) + print('--') + + assert 'Return value: 1' in result + #assert 'END_COM' in result + + +if __name__ == "__main__": + #test_example() + test_compile() diff --git a/tools/crosscompile.sh b/tools/crosscompile.sh index 57d4a2f..243e2ca 100644 --- a/tools/crosscompile.sh +++ b/tools/crosscompile.sh @@ -10,6 +10,8 @@ OPT=O3 mkdir -p $DEST +# -------------- Compile stencils -------------- + # Windows x86_64 (ARM64) python3 stencils/generate_stencils.py --abi ms $SRC gcc-13 -$OPT -c $SRC -o $DEST/stencils_AMD64_$OPT.o @@ -51,3 +53,9 @@ mipsel-linux-gnu-gcc-13 -$OPT -c $SRC -o $DEST/stencils_mipsel_$OPT.o # RISCV 64 Bit riscv64-linux-gnu-gcc-13 -$OPT -c $SRC -o $DEST/stencils_riscv64_$OPT.o + + +# -------------- Cross compile runner -------------- + +# Aarch64 +aarch64-linux-gnu-gcc-13 -static -O3 -o bin/coparun-aarch64 src/coparun/runmem.c src/coparun/coparun.c src/coparun/mem_man.c diff --git a/tools/extract_code.py b/tools/extract_code.py index 05496ad..13a916f 100644 --- a/tools/extract_code.py +++ b/tools/extract_code.py @@ -1,5 +1,16 @@ from copapy._binwrite import data_reader, Command, ByteOrder import argparse +from typing import Literal + +def patch(data: bytearray, offset: int, patch_mask: int, value: int, byteorder: Literal['little', 'big']) -> None: + # Read 4 bytes at the offset as a little-endian uint32 + original = int.from_bytes(data[offset:offset+4], byteorder) + + # Apply the patch + new_value = (original & ~patch_mask) | (value & patch_mask) + + # Write the new value back to the bytearray + data[offset:offset+4] = new_value.to_bytes(4, byteorder) if __name__ == "__main__": parser = argparse.ArgumentParser() @@ -14,8 +25,8 @@ if __name__ == "__main__": data_section_offset: int = args.data_section_offset byteorder: ByteOrder = args.byteorder - with open(input_file, mode='rb') as f: - dr = data_reader(f.read(), byteorder) + with open(input_file, mode='rb') as f_in: + dr = data_reader(f_in.read(), byteorder) buffer_index: int = 0 end_flag: int = 0 @@ -46,15 +57,13 @@ if __name__ == "__main__": offs = dr.read_int() mask = dr.read_int() value = dr.read_int(signed=True) - assert mask == 0xFFFFFFFF - program_data[offs:offs + 4] = value.to_bytes(4, byteorder, signed=True) + patch(program_data, offs, mask, value, byteorder) print(f"PATCH_FUNC patch_offs={offs} mask=0x{mask:x} value={value}") elif com == Command.PATCH_OBJECT: offs = dr.read_int() mask = dr.read_int() value = dr.read_int(signed=True) - assert mask == 0xFFFFFFFF - program_data[offs:offs + 4] = (value + data_section_offset).to_bytes(4, byteorder, signed=True) + patch(program_data, offs, mask, value + data_section_offset, byteorder) print(f"PATCH_OBJECT patch_offs={offs} mask=ox{mask:x} value={value}") elif com == Command.ENTRY_POINT: rel_entr_point = dr.read_int() @@ -73,7 +82,7 @@ if __name__ == "__main__": else: assert False, f"Unknown command: {com}" - with open(output_file, mode='wb') as f: - f.write(program_data) + with open(output_file, mode='wb') as f_out: + f_out.write(program_data) print(f"Code written to {output_file}.") diff --git a/tools/get_binaries.py b/tools/get_binaries.py index 0ebd8ee..74c1b60 100644 --- a/tools/get_binaries.py +++ b/tools/get_binaries.py @@ -30,6 +30,8 @@ def main() -> None: dest = 'bin' elif name == 'coparun' and os.name == 'posix': dest = 'bin' + elif name.startswith('coparun-'): + dest = 'bin' else: dest = '' diff --git a/tools/make_example.py b/tools/make_example.py index 6b47378..49787f3 100644 --- a/tools/make_example.py +++ b/tools/make_example.py @@ -1,5 +1,5 @@ from copapy import variable -from copapy.backend import Write, compile_to_dag +from copapy.backend import Write, compile_to_dag, stencil_db_from_package import copapy as cp from copapy._binwrite import Command @@ -40,5 +40,43 @@ def test_compile() -> None: dw.to_file('bin/test.copapy') +def test_compile_aarch64() -> None: + """Test compilation of a simple program.""" + c1 = variable(9.0) + + #ret = [c1 / 4, c1 / -4, c1 // 4, c1 // -4, (c1 * -1) // 4] + ret = [c1 // 3.3 + 5] + #ret = [cp.sqrt(c1)] + #c2 = cp._math.get_42() + #ret = [c2] + + out = [Write(r) for r in ret] + + sdb = stencil_db_from_package('aarch64') + dw, vars = compile_to_dag(out, sdb) + + # run program command + dw.write_com(Command.RUN_PROG) + + # read first 32 byte + dw.write_com(Command.READ_DATA) + dw.write_int(0) + dw.write_int(32) + + # read variables + for addr, lengths, _ in vars.values(): + dw.write_com(Command.READ_DATA) + dw.write_int(addr) + dw.write_int(lengths) + + dw.write_com(Command.END_COM) + + print('* Data to runner:') + dw.print() + + dw.to_file('bin/test-aarch64.copapy') + + if __name__ == "__main__": test_compile() + test_compile_aarch64()