CI: exclude runner for arm thumb

.github/workflows/ci.yml

@@ -151,7 +151,7 @@ jobs:
     - name: Use ARM64 container
       run: |
         docker run --rm -v $PWD:/app -w /app --platform linux/arm64 ghcr.io/nonannet/arm64_test:1 \
-        bash -lc "pip install . && \
+        bash -lc "pip install .[mindev] && \
                   mkdir -p build/runner && \
                   gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \
                   src/coparun/coparun.c src/coparun/mem_man.c && \
@@ -180,7 +180,7 @@ jobs:
     - name: Use ARMv6 container
       run: |
         docker run --rm -v $PWD:/app -w /app --platform linux/arm/v6 ghcr.io/nonannet/armv6_test:1 \
-        bash -lc "pip install . && \
+        bash -lc "pip install .[mindev] && \
                   mkdir -p build/runner && \
                   gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \
                   src/coparun/coparun.c src/coparun/mem_man.c && \
@@ -209,15 +209,15 @@ jobs:
     - name: Use ARMv7 container
       run: |
         docker run --rm -v $PWD:/app -w /app --platform linux/arm/v7 ghcr.io/nonannet/armv7_test:1 \
-        bash -lc "pip install . && \
+        bash -lc "pip install .[mindev] && \
                   mkdir -p build/runner && \
-                  gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \
+                  gcc  -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \
+                  -Wall -Wextra -Wconversion -Wsign-conversion \
+                  -Wshadow -Wstrict-overflow -O3 \
+                  -DENABLE_LOGGING \
+                  -o build/runner/coparun src/coparun/runmem.c \
                   src/coparun/coparun.c src/coparun/mem_man.c && \
                   pytest && \
-                  export CP_TARGET_ARCH=armv7thumb && \
-                  pytest && \
-                  export CP_TARGET_ARCH=armv7mthumb && \
-                  pytest && \
                   bash tools/create_asm.sh"
 
     - uses: actions/upload-artifact@v4
@@ -225,6 +225,74 @@ jobs:
         name: runner-linux-armv7
         path: build/runner/*
 
+  build-armv7thumb:
+    needs: [build_stencils]
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/download-artifact@v4
+      with:
+        name: stencil-object-files
+        path: src/copapy/obj
+    - name: Set up QEMU for ARMv7
+      uses: docker/setup-qemu-action@v3
+      with:
+        platforms: linux/arm/v7
+    - name: Use ARMv7 container
+      run: |
+        docker run --rm -v $PWD:/app -w /app --platform linux/arm/v7 ghcr.io/nonannet/armv7_test:1 \
+        bash -lc "pip install .[mindev] && \
+                  mkdir -p build/runner && \
+                  gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \
+                  -Wall -Wextra -Wconversion -Wsign-conversion \
+                  -Wshadow -Wstrict-overflow -O3 \
+                  -DENABLE_LOGGING \
+                  -o build/runner/coparun src/coparun/runmem.c \
+                  src/coparun/coparun.c src/coparun/mem_man.c && \
+                  export CP_TARGET_ARCH=armv7thumb && \
+                  pytest -m 'not runner' && \
+                  bash tools/create_asm.sh"
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: runner-linux-armv7thumb
+        path: build/runner/*
+
+  build-armv7mthumb:
+    needs: [build_stencils]
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/download-artifact@v4
+      with:
+        name: stencil-object-files
+        path: src/copapy/obj
+    - name: Set up QEMU for ARMv7
+      uses: docker/setup-qemu-action@v3
+      with:
+        platforms: linux/arm/v7
+    - name: Use ARMv7 container
+      run: |
+        docker run --rm -v $PWD:/app -w /app --platform linux/arm/v7 ghcr.io/nonannet/armv7_test:1 \
+        bash -lc "pip install .[mindev] && \
+                  mkdir -p build/runner && \
+                  gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \
+                  -Wall -Wextra -Wconversion -Wsign-conversion \
+                  -Wshadow -Wstrict-overflow -O3 \
+                  -DENABLE_LOGGING \
+                  -o build/runner/coparun src/coparun/runmem.c \
+                  src/coparun/coparun.c src/coparun/mem_man.c && \
+                  export CP_TARGET_ARCH=armv7mthumb && \
+                  pytest -m 'not runner' && \
+                  bash tools/create_asm.sh"
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: runner-linux-armv7mthumb
+        path: build/runner/*
+
   build-windows:
     needs: [build_stencils]
     runs-on: windows-latest
@@ -248,7 +316,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install Python dependencies
-      run: python -m pip install .[dev]
+      run: python -m pip install .[mindev]
 
     - name: Set up MSVC environment
       uses: microsoft/setup-msbuild@v2
@@ -307,6 +375,8 @@ jobs:
           cp tmp/runner-linux-arm64/coparun release/coparun-aarch64
           cp tmp/runner-linux-armv6/coparun release/coparun-armv6
           cp tmp/runner-linux-armv7/coparun release/coparun-armv7
+          cp tmp/runner-linux-armv7thumb/coparun release/coparun-armv7thumb
+          cp tmp/runner-linux-armv7mthumb/coparun release/coparun-armv7mthumb
           cp tmp/runner-win/coparun*.exe release/
 
           TAG="${{ steps.version.outputs.version }}"

pyproject.toml

@@ -45,14 +45,18 @@ dev = [
     "ruff",
     "mypy",
     "pytest",
-    "pelfy>=1.0.7"
+    "pelfy>=1.0.8"
+]
+mindev = [
+    "pytest",
+    "pelfy>=1.0.8"
 ]
 doc_build = [
     "sphinx",
     "pydata_sphinx_theme",
     "sphinx-autodoc-typehints",
     "myst-parser",
-  "pelfy>=1.0.7"
+    "pelfy>=1.0.8"
 ]
 
 [tool.mypy]

src/copapy/_binwrite.py

@@ -25,7 +25,7 @@ class data_writer():
         self._data: list[tuple[str, bytes, int]] = []
         self.byteorder: ByteOrder = byteorder
 
-    def copy(self):
+    def copy(self) -> 'data_writer':
         cp = data_writer(self.byteorder)
         cp._data = self._data.copy()
         return cp

src/copapy/_stencils.py

@@ -135,6 +135,11 @@ def get_op_after_last_call_in_function(func: pelfy.elf_symbol) -> int:
     return reloc.fields['r_offset'] - func.offset_in_section + 4
 
 
+def add_sign_int32(value: int) -> int:
+    """Convert a 32-bit unsigned integer to a signed integer."""
+    return value - 0x100000000 if value > 0x7FFFFFFF else value
+
+
 class stencil_database():
     """A class for loading and querying a stencil database from an ELF object file
 
@@ -226,6 +231,7 @@ class stencil_database():
                 cache.append(reloc_entry)
                 yield reloc_entry
 
+
     def get_patch(self, relocation: relocation_entry, symbol_address: int, function_offset: int, symbol_type: int) -> patch_entry:
         """Return patch positions for a provided symbol (function or object)
 
@@ -251,12 +257,14 @@ class stencil_database():
 
         if pr.type.endswith('64_PC32') or pr.type.endswith('64_PLT32'):
             # S + A - P
-            patch_value = symbol_address + pr.fields['r_addend'] - patch_offset
+            addend = add_sign_int32(pr.fields['r_addend'])
+            patch_value = symbol_address + addend - patch_offset
             #print(f" *> {pr.type} {patch_value=} {symbol_address=} {pr.fields['r_addend']=} {pr.bits=}, {function_offset=} {patch_offset=}")
 
         elif pr.type == 'R_386_PC32':
             # S + A - P
-            patch_value = symbol_address + pr.fields['r_addend'] - patch_offset
+            addend = add_sign_int32(pr.fields['r_addend'])
+            patch_value = symbol_address + addend - patch_offset
             #print(f" *> {pr.type}     {pr.symbol.name} {patch_value=} {symbol_address=} {pr.fields['r_addend']=} {bin(pr.fields['r_addend'])} {pr.bits=}, {function_offset=} {patch_offset=}")
 
         elif pr.type == 'R_386_32':

tests/test_ops_armv7mthumb.py

@@ -0,0 +1,174 @@
+from copapy import NumLike, iif, value
+from copapy.backend import Store, compile_to_dag, add_read_value_remote
+import subprocess
+from copapy import _binwrite
+import copapy.backend as backend
+import os
+import warnings
+import re
+import struct
+import pytest
+import copapy as cp
+
+if os.name == 'nt':
+    # On Windows wsl and qemu-user is required:
+    # sudo apt install qemu-user
+    qemu_command = ['wsl', 'qemu-arm']
+else:
+    qemu_command = ['qemu-arm']
+
+
+def parse_results(log_text: str) -> dict[int, bytes]:
+    regex = r"^READ_DATA offs=(\d*) size=(\d*) data=(.*)$"
+    matches = re.finditer(regex, log_text, re.MULTILINE)
+    var_dict: dict[int, bytes] = {}
+
+    for match in matches:
+        value_str: list[str] = match.group(3).strip().split(' ')
+        #print('--', value_str)
+        value = bytes(int(v, base=16) for v in value_str)
+        if len(value) <= 8:
+            var_dict[int(match.group(1))] = value
+
+    return var_dict
+
+
+def run_command(command: list[str]) -> str:
+    result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8', check=False)
+    assert result.returncode != 11, f"SIGSEGV (segmentation fault)\n -Error occurred: {result.stderr}\n -Output: {result.stdout}"
+    assert result.returncode == 0, f"\n -Error occurred: {result.stderr}\n -Output: {result.stdout}"
+    return result.stdout
+
+
+def check_for_qemu() -> bool:
+    command = qemu_command + ['--version']
+    try:
+        result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
+    except Exception:
+        return False
+    return result.returncode == 0
+
+
+def function1(c1: NumLike) -> list[NumLike]:
+    return [c1 / 4, c1 / -4, c1 // 4, c1 // -4, (c1 * -1) // 4,
+            c1 * 4, c1 * -4,
+            c1 + 4, c1 - 4,
+            c1 > 2, c1 > 100, c1 < 4, c1 < 100]
+
+def function1ex(c1: NumLike) -> list[NumLike]:
+    return [c1 // 4]
+
+
+def function2(c1: NumLike) -> list[NumLike]:
+    return [c1 * 4.44, c1 * -4.44]
+
+
+def function3(c1: NumLike) -> list[NumLike]:
+    return [c1 / 4]
+
+
+def function4(c1: NumLike) -> list[NumLike]:
+    return [c1 == 9, c1 == 4, c1 != 9, c1 != 4]
+
+
+def function5(c1: NumLike) -> list[NumLike]:
+    return [c1 == True, c1 == False, c1 != True, c1 != False, c1 / 2, c1 + 2]
+
+
+def function6(c1: NumLike) -> list[NumLike]:
+    return [c1 == True]
+
+
+def iiftests(c1: NumLike) -> list[NumLike]:
+    return [iif(c1 > 5, 8, 9),
+            iif(c1 < 5, 8.5, 9.5),
+            iif(1 > 5, 3.3, 8.8) + c1,
+            iif(1 < 5, c1 * 3.3, 8.8),
+            iif(c1 < 5, c1 * 3.3, 8.8)]
+
+
+@pytest.mark.runner
+def test_compile():
+    c_i = value(9)
+    c_f = value(1.111)
+    c_b = value(True)
+
+    ret_test = function1(c_i) + function1(c_f) + function2(c_i) + function2(c_f) + function3(c_i) + function4(c_i) + function5(c_b) + [value(9) % 2] + iiftests(c_i) + iiftests(c_f) + [cp.asin(c_i/10)]
+    ret_ref = function1(9) + function1(1.111) + function2(9) + function2(1.111) + function3(9) + function4(9) + function5(True) + [9 % 2] + iiftests(9) + iiftests(1.111) + [cp.asin(9/10)]
+
+    out = [Store(r) for r in ret_test]
+
+    sdb = backend.stencil_db_from_package('armv7mthumb')
+    dw, variables = compile_to_dag(out, sdb)
+
+    #dw.write_com(_binwrite.Command.READ_DATA)
+    #dw.write_int(0)
+    #dw.write_int(28)
+
+    du = dw.copy()
+    dw.write_com(_binwrite.Command.RUN_PROG)
+    du.write_com(_binwrite.Command.DUMP_CODE)
+
+    for v in ret_test:
+        assert isinstance(v, value)
+        add_read_value_remote(dw, variables, v.net)
+
+    #dw.write_com(_binwrite.Command.READ_DATA)
+    #dw.write_int(0)
+    #dw.write_int(28)
+
+    dw.write_com(_binwrite.Command.END_COM)
+    du.write_com(_binwrite.Command.END_COM)
+
+    #print('* Data to runner:')
+    #dw.print()
+
+    dw.to_file('build/runner/test-armv7mthumb.copapy')
+    du.to_file('build/runner/test-armv7mthumb-dump.copapy')
+
+    if not check_for_qemu():
+        warnings.warn("qemu-armv7 not found, armv7 test skipped!", UserWarning)
+        return
+    if not os.path.isfile('build/runner/coparun-armv7'):
+        warnings.warn("armv7 runner not found, armv7 test skipped!", UserWarning)
+        return
+
+    print('----- Dump code...')
+    command = qemu_command + ['build/runner/coparun-armv7', 'build/runner/test-armv7mthumb-dump.copapy', 'build/runner/test.copapy-armv7mthumb.bin']
+    result = run_command(command)
+
+    print('----- Run code...')
+    command = qemu_command + ['build/runner/coparun-armv7', 'build/runner/test-armv7mthumb.copapy']
+    result = run_command(command)
+
+
+    print('* Output from runner:\n--')
+    print(result)
+    print('--')
+
+    assert 'Return value: 1' in result
+
+    result_data = parse_results(result)
+
+    for test, ref in zip(ret_test, ret_ref):
+        assert isinstance(test, value)
+        address = variables[test.net][0]
+        data = result_data[address]
+        if test.dtype == 'int':
+            val = int.from_bytes(data, sdb.byteorder, signed=True)
+        elif test.dtype == 'bool':
+            val = bool.from_bytes(data, sdb.byteorder)
+        elif test.dtype == 'float':
+            en = {'little': '<', 'big': '>'}[sdb.byteorder]
+            val = struct.unpack(en + 'f', data)[0]
+            assert isinstance(val, float)
+        else:
+            raise Exception(f"Unknown type: {test.dtype}")
+        print('+', val, ref, test.dtype, f"  addr={address}")
+        for t in (int, float, bool):
+            assert isinstance(val, t) == isinstance(ref, t), f"Result type does not match for {val} and {ref}"
+            assert val == pytest.approx(ref, 1e-5), f"Result does not match: {val} and reference: {ref}"  # pyright: ignore[reportUnknownMemberType]
+
+
+if __name__ == "__main__":
+    test_compile()

tests/test_ops_armv7thumb.py

@@ -98,7 +98,7 @@ def test_compile():
 
     out = [Store(r) for r in ret_test]
 
-    sdb = backend.stencil_db_from_package('armv7mthumb')
+    sdb = backend.stencil_db_from_package('armv7thumb')
     dw, variables = compile_to_dag(out, sdb)
 
     #dw.write_com(_binwrite.Command.READ_DATA)
@@ -129,16 +129,16 @@ def test_compile():
     if not check_for_qemu():
         warnings.warn("qemu-armv7 not found, armv7 test skipped!", UserWarning)
         return
-    if not os.path.isfile('build/runner/coparun-armv7thumb'):
+    if not os.path.isfile('build/runner/coparun-armv7'):
-        warnings.warn("armv7thumb runner not found, armv7thumb test skipped!", UserWarning)
+        warnings.warn("armv7 runner not found, armv7 test skipped!", UserWarning)
         return
 
     print('----- Dump code...')
-    command = qemu_command + ['build/runner/coparun-armv7thumb', 'build/runner/test-armv7thumb-dump.copapy', 'build/runner/test.copapy-armv7thumb.bin']
+    command = qemu_command + ['build/runner/coparun-armv7', 'build/runner/test-armv7thumb-dump.copapy', 'build/runner/test.copapy-armv7thumb.bin']
     result = run_command(command)
 
     print('----- Run code...')
-    command = qemu_command + ['build/runner/coparun-armv7thumb', 'build/runner/test-armv7thumb.copapy']
+    command = qemu_command + ['build/runner/coparun-armv7', 'build/runner/test-armv7thumb.copapy']
     result = run_command(command)
 
 

tests/test_stencil_db.py

@@ -19,11 +19,18 @@ def test_start_end_function():
 
         if symbol.relocations and symbol.relocations[-1].symbol.info == 'STT_NOTYPE':
 
-            print('-', sym_name, get_stencil_position(symbol), len(symbol.data))
+            if symbol.section and symbol.section.name == '.text':
+                print('SKIP', sym_name, '(Aux function, not a stencil)')
+                continue
+
+            if symbol.section:
+                function_size = symbol.section.fields['sh_size']  # len(symbol.data) excludes nop after the function
+
+                print('-', sym_name, get_stencil_position(symbol), function_size)
 
                 start, end = get_stencil_position(symbol)
 
-            assert start >= 0 and end >= start and end <= len(symbol.data)
+                assert (start >= 0 and end >= start and end <= function_size)
 
 
 def test_aux_functions():

tools/build.sh

@@ -128,6 +128,7 @@ if [[ "$ARCH" == "arm-v7" || "$ARCH" == "all" ]]; then
         $DEST/stencils_armv7_O3.o \
         > build/stencils/stencils_armv7_O3.asm
 
+    # The same runner for all ARM7
     arm-linux-gnueabihf-gcc \
         -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \
         -Wall -Wextra -Wconversion -Wsign-conversion \
@@ -161,8 +162,9 @@ if [[ "$ARCH" == "arm-v7-thumb" || "$ARCH" == "all" ]]; then
         $DEST/stencils_armv7thumb_O3.o \
         > build/stencils/stencils_armv7thumb_O3.asm
 
+    # The same runner for all ARM7
     arm-linux-gnueabihf-gcc \
-        -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb -static \
+        -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -static \
         -Wall -Wextra -Wconversion -Wsign-conversion \
         -Wshadow -Wstrict-overflow -O3 \
         -DENABLE_LOGGING \
@@ -194,8 +196,9 @@ if [[ "$ARCH" == "arm-v7m-thumb" || "$ARCH" == "all" ]]; then
         $DEST/stencils_armv7mthumb_O3.o \
         > build/stencils/stencils_armv7mthumb_O3.asm
 
+    # The same runner for all ARM7
     arm-linux-gnueabihf-gcc \
-        -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb -static \
+        -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -static \
         -Wall -Wextra -Wconversion -Wsign-conversion \
         -Wshadow -Wstrict-overflow -O3 \
         -DENABLE_LOGGING \

tools/crosscompile.sh

@@ -43,12 +43,12 @@ arm-none-eabi-ld -r $STMP /object_files/musl_objects_armv7.o $LIBGCC -o $DEST/st
 
 # ARMv7 Thumb for Cortex-A with hardware fp
 arm-none-eabi-gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb $FLAGS -$OPT -c $SRC -o $STMP
-LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name)
+LIBGCC=$(arm-none-eabi-gcc -march=armv7 -mfpu=vfp3 -mthumb -print-libgcc-file-name)
 arm-none-eabi-ld -r $STMP /object_files/musl_objects_armv7thumb.o $LIBGCC -o $DEST/stencils_armv7thumb_$OPT.o
 
 # Armv7 Thumb for Cortex-M3..7 hardware fp
 arm-none-eabi-gcc -march=armv7e-m -mfpu=fpv4-sp-d16 -mfloat-abi=hard -mthumb $FLAGS -$OPT -c $SRC -o $STMP
-LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name)
+LIBGCC=$(arm-none-eabi-gcc -march=armv7e-m -mfpu=fpv4-sp-d16 -mfloat-abi=hard -mthumb -print-libgcc-file-name)
 arm-none-eabi-ld -r $STMP /object_files/musl_objects_armv7mthumb.o $LIBGCC -o $DEST/stencils_armv7mthumb_$OPT.o
 
 # PowerPC64LE