Cross compilation for aarch64-runner added

.github/workflows/build_wheels.yml

@@ -23,6 +23,7 @@ jobs:
           path: src/copapy/obj/*.o
 
   build_wheels:
+    if: contains(github.ref, '-beta') == false
     needs: [build_stencils]
     runs-on: ${{ matrix.os }}
     strategy:
@@ -65,6 +66,7 @@ jobs:
           path: wheelhouse/*.whl
 
 #  publish:
+#    if: contains(github.ref, '-beta') == false
 #    needs: [build_wheels]
 #    runs-on: ubuntu-latest
 #    steps:

.github/workflows/ci.yml

@@ -25,6 +25,11 @@ jobs:
           name: stencil-object-files
           path: src/copapy/obj/*.o
 
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cross-runner
+          path: bin/coparun-*
+
   build-ubuntu:
     needs: [build_stencils]
     runs-on: ubuntu-latest
@@ -71,6 +76,11 @@ jobs:
         echo '<p>test.copapy.asm</p>' >> $GITHUB_STEP_SUMMARY
         python tools/clean_asm.py bin/test.copapy.asm >> $GITHUB_STEP_SUMMARY
 
+        python tools/extract_code.py "bin/test-aarch64.copapy" "bin/test-aarch64.copapy.bin"
+        aarch64-linux-gnu-objdump -D -b binary -m aarch64 --adjust-vma=0x1000 bin/test-aarch64.copapy.bin > bin/test-aarch64.copapy.asm
+        echo '<p>test-aarch64.copapy.asm</p>' >> $GITHUB_STEP_SUMMARY
+        python tools/clean_asm.py bin/test-aarch64.copapy.asm >> $GITHUB_STEP_SUMMARY
+
         objdump -d -x src/copapy/obj/stencils_x86_64_O3.o > bin/stencils_x86_64_O3.asm
         echo '<p>stencils_x86_64_O3.asm</p>' >> $GITHUB_STEP_SUMMARY
         python tools/clean_asm.py bin/stencils_x86_64_O3.asm >> $GITHUB_STEP_SUMMARY
@@ -176,6 +186,7 @@ jobs:
           set -v
           mkdir -p release
           cp tmp/stencil-object-files/*.o release/
+          cp tmp/cross-runner/coparun-* release/
           cp tmp/runner-linux/coparun release/
           cp tmp/runner-win/coparun.exe release/
 

src/copapy/_stencils.py

@@ -27,12 +27,13 @@ class patch_entry:
 
 
 def translate_relocation(reloc: pelfy.elf_relocation, offset: int) -> patch_entry:
-    if reloc.type in ('R_AMD64_PLT32', 'R_AMD64_PC32'):
+    if reloc.type.endswith('_PLT32') or reloc.type.endswith('_PC32'):
         # S + A - P
         mask = 0xFFFFFFFF  # 32 bit
         imm = offset
 
-    elif reloc.type.endswith('_JUMP26'):
+    elif reloc.type.endswith('_JUMP26') or reloc.type.endswith('_CALL26'):
+        # S + A - P
         assert reloc.file.byteorder == 'little', "Big endian not supported for ARM64"
         mask = 0x3ffffff  # 26 bit
         imm = offset >> 2

src/copapy/backend.py

@@ -1,5 +1,5 @@
 from ._target import add_read_command
-from ._basic_types import Net, Op, Node, CPConstant, Write
+from ._basic_types import Net, Op, Node, CPConstant, Write, stencil_db_from_package
 from ._compiler import compile_to_dag, \
     stable_toposort, get_const_nets, get_all_dag_edges, add_read_ops, \
     add_write_ops
@@ -17,4 +17,5 @@ __all__ = [
     "get_all_dag_edges",
     "add_read_ops",
     "add_write_ops",
+    "stencil_db_from_package"
 ]

tests/test_compile_aarch64.py

@@ -0,0 +1,88 @@
+from copapy import variable, NumLike
+from copapy.backend import Write, compile_to_dag, add_read_command
+import subprocess
+import struct
+from copapy import _binwrite
+import copapy.backend as backend
+import os
+import pytest
+
+if os.name == 'nt':
+    # On Windows wsl and qemu-user is required:
+    # sudo apt install qemu-user
+    qemu_command = ['wsl', 'qemu-aarch64', 'bin/coparun-aarch64', 'bin/test.copapy']
+else:
+    qemu_command = ['qemu-aarch64', 'bin/coparun-aarch64', 'bin/test.copapy']
+
+
+def run_command(command: list[str]) -> str:
+    result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8', check=False)
+    assert result.returncode != 11, f"SIGSEGV (segmentation fault)\n -Error occurred: {result.stderr}\n -Output: {result.stdout}"
+    assert result.returncode == 0, f"\n -Error occurred: {result.stderr}\n -Output: {result.stdout}"
+    return result.stdout
+
+
+def test_example():
+    c1 = 4
+    c2 = 2
+
+    i1 = c1 * 2
+    r1 = i1 + 7 + (c2 + 7 * 9)
+    r2 = i1 + 9
+
+    en = {'little': '<', 'big': '>'}['little']
+    data = struct.pack(en + 'i', r1)
+    print("example r1 " + ' '.join(f'{b:02X}' for b in data))
+
+    data = struct.pack(en + 'i', r2)
+    print("example r2 " + ' '.join(f'{b:02X}' for b in data))
+
+
+def function(c1: NumLike, c2: NumLike) -> tuple[NumLike, ...]:
+    i1 = c1 // 3.3 + 5
+    i2 = c2 * 5 + c1
+    r1 = i1 + i2 * 55 / 4
+    r2 = 4 * i2 + 5
+
+    return i1, i2, r1, r2
+
+
+@pytest.mark.skip(reason="no way of currently testing this")
+def test_compile():
+    c1 = variable(4)
+    c2 = variable(2)
+
+    ret = function(c1, c2)
+    #ret = [c1 // 3.3 + 5]
+
+    out = [Write(r) for r in ret]
+
+    sdb = backend.stencil_db_from_package('aarch64', 'O3')
+    il, variables = compile_to_dag(out, sdb)
+
+    # run program command
+    il.write_com(_binwrite.Command.RUN_PROG)
+
+    for net in ret:
+        assert isinstance(net, backend.Net)
+        add_read_command(il, variables, net)
+
+    il.write_com(_binwrite.Command.END_COM)
+
+    print('* Data to runner:')
+    il.print()
+
+    il.to_file('bin/test.copapy')
+
+    result = run_command(qemu_command)
+    print('* Output from runner:\n--')
+    print(result)
+    print('--')
+
+    assert 'Return value: 1' in result
+    #assert 'END_COM' in result
+
+
+if __name__ == "__main__":
+    #test_example()
+    test_compile()

tools/crosscompile.sh

@@ -10,6 +10,8 @@ OPT=O3
 
 mkdir -p $DEST
 
+# -------------- Compile stencils --------------
+
 # Windows x86_64 (ARM64)
 python3 stencils/generate_stencils.py --abi ms $SRC
 gcc-13 -$OPT -c $SRC -o $DEST/stencils_AMD64_$OPT.o
@@ -51,3 +53,9 @@ mipsel-linux-gnu-gcc-13 -$OPT -c $SRC -o $DEST/stencils_mipsel_$OPT.o
 
 # RISCV 64 Bit
 riscv64-linux-gnu-gcc-13 -$OPT -c $SRC -o $DEST/stencils_riscv64_$OPT.o
+
+
+# -------------- Cross compile runner --------------
+
+# Aarch64
+aarch64-linux-gnu-gcc-13 -static -O3 -o bin/coparun-aarch64 src/coparun/runmem.c src/coparun/coparun.c src/coparun/mem_man.c

tools/extract_code.py

@@ -1,5 +1,16 @@
 from copapy._binwrite import data_reader, Command, ByteOrder
 import argparse
+from typing import Literal
+
+def patch(data: bytearray, offset: int, patch_mask: int, value: int, byteorder: Literal['little', 'big']) -> None:
+    # Read 4 bytes at the offset as a little-endian uint32
+    original = int.from_bytes(data[offset:offset+4], byteorder)
+    
+    # Apply the patch
+    new_value = (original & ~patch_mask) | (value & patch_mask)
+    
+    # Write the new value back to the bytearray
+    data[offset:offset+4] = new_value.to_bytes(4, byteorder)
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
@@ -14,8 +25,8 @@ if __name__ == "__main__":
     data_section_offset: int = args.data_section_offset
     byteorder: ByteOrder = args.byteorder
 
-    with open(input_file, mode='rb') as f:
-        dr = data_reader(f.read(), byteorder)
+    with open(input_file, mode='rb') as f_in:
+        dr = data_reader(f_in.read(), byteorder)
 
     buffer_index: int = 0
     end_flag: int = 0
@@ -46,15 +57,13 @@ if __name__ == "__main__":
             offs = dr.read_int()
             mask = dr.read_int()
             value = dr.read_int(signed=True)
-            assert mask == 0xFFFFFFFF
-            program_data[offs:offs + 4] = value.to_bytes(4, byteorder, signed=True)
+            patch(program_data, offs, mask, value, byteorder)
             print(f"PATCH_FUNC patch_offs={offs} mask=0x{mask:x} value={value}")
         elif com == Command.PATCH_OBJECT:
             offs = dr.read_int()
             mask = dr.read_int()
             value = dr.read_int(signed=True)
-            assert mask == 0xFFFFFFFF
-            program_data[offs:offs + 4] = (value + data_section_offset).to_bytes(4, byteorder, signed=True)
+            patch(program_data, offs, mask, value + data_section_offset, byteorder)
             print(f"PATCH_OBJECT patch_offs={offs} mask=ox{mask:x} value={value}")
         elif com == Command.ENTRY_POINT:
             rel_entr_point = dr.read_int()
@@ -73,7 +82,7 @@ if __name__ == "__main__":
         else:
             assert False, f"Unknown command: {com}"
 
-    with open(output_file, mode='wb') as f:
-        f.write(program_data)
+    with open(output_file, mode='wb') as f_out:
+        f_out.write(program_data)
 
     print(f"Code written to {output_file}.")

tools/get_binaries.py

@@ -30,6 +30,8 @@ def main() -> None:
             dest = 'bin'
         elif name == 'coparun' and os.name == 'posix':
             dest = 'bin'
+        elif name.startswith('coparun-'):
+            dest = 'bin'
         else:
             dest = ''
 

tools/make_example.py

@@ -1,5 +1,5 @@
 from copapy import variable
-from copapy.backend import Write, compile_to_dag
+from copapy.backend import Write, compile_to_dag, stencil_db_from_package
 import copapy as cp
 from copapy._binwrite import Command
 
@@ -40,5 +40,43 @@ def test_compile() -> None:
     dw.to_file('bin/test.copapy')
 
 
+def test_compile_aarch64() -> None:
+    """Test compilation of a simple program."""
+    c1 = variable(9.0)
+
+    #ret = [c1 / 4, c1 / -4, c1 // 4, c1 // -4, (c1 * -1) // 4]
+    ret = [c1 // 3.3 + 5]
+    #ret = [cp.sqrt(c1)]
+    #c2 = cp._math.get_42()
+    #ret = [c2]
+
+    out = [Write(r) for r in ret]
+
+    sdb = stencil_db_from_package('aarch64')
+    dw, vars = compile_to_dag(out, sdb)
+
+    # run program command
+    dw.write_com(Command.RUN_PROG)
+
+    # read first 32 byte
+    dw.write_com(Command.READ_DATA)
+    dw.write_int(0)
+    dw.write_int(32)
+
+    # read variables
+    for addr, lengths, _ in vars.values():
+        dw.write_com(Command.READ_DATA)
+        dw.write_int(addr)
+        dw.write_int(lengths)
+
+    dw.write_com(Command.END_COM)
+
+    print('* Data to runner:')
+    dw.print()
+
+    dw.to_file('bin/test-aarch64.copapy')
+
+
 if __name__ == "__main__":
     test_compile()
+    test_compile_aarch64()