Merge a924d42e6a into e52cbe9e1b

.github/workflows/ci.yml

@@ -151,7 +151,7 @@ jobs:
     - name: Use ARM64 container
       run: |
         docker run --rm -v $PWD:/app -w /app --platform linux/arm64 ghcr.io/nonannet/arm64_test:1 \
-        bash -lc "pip install .[mindev] && \
+        bash -lc "pip install . && \
                   mkdir -p build/runner && \
                   gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \
                   src/coparun/coparun.c src/coparun/mem_man.c && \
@@ -180,7 +180,7 @@ jobs:
     - name: Use ARMv6 container
       run: |
         docker run --rm -v $PWD:/app -w /app --platform linux/arm/v6 ghcr.io/nonannet/armv6_test:1 \
-        bash -lc "pip install .[mindev] && \
+        bash -lc "pip install . && \
                   mkdir -p build/runner && \
                   gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \
                   src/coparun/coparun.c src/coparun/mem_man.c && \
@@ -209,15 +209,15 @@ jobs:
     - name: Use ARMv7 container
       run: |
         docker run --rm -v $PWD:/app -w /app --platform linux/arm/v7 ghcr.io/nonannet/armv7_test:1 \
-        bash -lc "pip install .[mindev] && \
+        bash -lc "pip install . && \
                   mkdir -p build/runner && \
-                  gcc  -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \
+                  gcc -O3 -static -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \
-                  -Wall -Wextra -Wconversion -Wsign-conversion \
-                  -Wshadow -Wstrict-overflow -O3 \
-                  -DENABLE_LOGGING \
-                  -o build/runner/coparun src/coparun/runmem.c \
                   src/coparun/coparun.c src/coparun/mem_man.c && \
                   pytest && \
+                  export CP_TARGET_ARCH=armv7thumb && \
+                  pytest && \
+                  export CP_TARGET_ARCH=armv7mthumb && \
+                  pytest && \
                   bash tools/create_asm.sh"
 
     - uses: actions/upload-artifact@v4
@@ -225,74 +225,6 @@ jobs:
         name: runner-linux-armv7
         path: build/runner/*
 
-  build-armv7thumb:
-    needs: [build_stencils]
-    runs-on: ubuntu-latest
-    continue-on-error: true
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/download-artifact@v4
-      with:
-        name: stencil-object-files
-        path: src/copapy/obj
-    - name: Set up QEMU for ARMv7
-      uses: docker/setup-qemu-action@v3
-      with:
-        platforms: linux/arm/v7
-    - name: Use ARMv7 container
-      run: |
-        docker run --rm -v $PWD:/app -w /app --platform linux/arm/v7 ghcr.io/nonannet/armv7_test:1 \
-        bash -lc "pip install .[mindev] && \
-                  mkdir -p build/runner && \
-                  gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \
-                  -Wall -Wextra -Wconversion -Wsign-conversion \
-                  -Wshadow -Wstrict-overflow -O3 \
-                  -DENABLE_LOGGING \
-                  -o build/runner/coparun src/coparun/runmem.c \
-                  src/coparun/coparun.c src/coparun/mem_man.c && \
-                  export CP_TARGET_ARCH=armv7thumb && \
-                  pytest -m 'not runner' && \
-                  bash tools/create_asm.sh"
-
-    - uses: actions/upload-artifact@v4
-      with:
-        name: runner-linux-armv7thumb
-        path: build/runner/*
-
-  build-armv7mthumb:
-    needs: [build_stencils]
-    runs-on: ubuntu-latest
-    continue-on-error: true
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/download-artifact@v4
-      with:
-        name: stencil-object-files
-        path: src/copapy/obj
-    - name: Set up QEMU for ARMv7
-      uses: docker/setup-qemu-action@v3
-      with:
-        platforms: linux/arm/v7
-    - name: Use ARMv7 container
-      run: |
-        docker run --rm -v $PWD:/app -w /app --platform linux/arm/v7 ghcr.io/nonannet/armv7_test:1 \
-        bash -lc "pip install .[mindev] && \
-                  mkdir -p build/runner && \
-                  gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \
-                  -Wall -Wextra -Wconversion -Wsign-conversion \
-                  -Wshadow -Wstrict-overflow -O3 \
-                  -DENABLE_LOGGING \
-                  -o build/runner/coparun src/coparun/runmem.c \
-                  src/coparun/coparun.c src/coparun/mem_man.c && \
-                  export CP_TARGET_ARCH=armv7mthumb && \
-                  pytest -m 'not runner' && \
-                  bash tools/create_asm.sh"
-
-    - uses: actions/upload-artifact@v4
-      with:
-        name: runner-linux-armv7mthumb
-        path: build/runner/*
-
   build-windows:
     needs: [build_stencils]
     runs-on: windows-latest
@@ -316,7 +248,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install Python dependencies
-      run: python -m pip install .[mindev]
+      run: python -m pip install .[dev]
 
     - name: Set up MSVC environment
       uses: microsoft/setup-msbuild@v2
@@ -375,8 +307,6 @@ jobs:
           cp tmp/runner-linux-arm64/coparun release/coparun-aarch64
           cp tmp/runner-linux-armv6/coparun release/coparun-armv6
           cp tmp/runner-linux-armv7/coparun release/coparun-armv7
-          cp tmp/runner-linux-armv7thumb/coparun release/coparun-armv7thumb
-          cp tmp/runner-linux-armv7mthumb/coparun release/coparun-armv7mthumb
           cp tmp/runner-win/coparun*.exe release/
 
           TAG="${{ steps.version.outputs.version }}"

pyproject.toml

@@ -45,18 +45,14 @@ dev = [
     "ruff",
     "mypy",
     "pytest",
-    "pelfy>=1.0.8"
+    "pelfy>=1.0.7"
-]
-mindev = [
-    "pytest",
-    "pelfy>=1.0.8"
 ]
 doc_build = [
   "sphinx",
   "pydata_sphinx_theme",
   "sphinx-autodoc-typehints",
   "myst-parser",
-    "pelfy>=1.0.8"
+  "pelfy>=1.0.7"
 ]
 
 [tool.mypy]

src/copapy/_binwrite.py

@@ -25,7 +25,7 @@ class data_writer():
         self._data: list[tuple[str, bytes, int]] = []
         self.byteorder: ByteOrder = byteorder
 
-    def copy(self) -> 'data_writer':
+    def copy(self):
         cp = data_writer(self.byteorder)
         cp._data = self._data.copy()
         return cp

src/copapy/_stencils.py

@@ -135,11 +135,6 @@ def get_op_after_last_call_in_function(func: pelfy.elf_symbol) -> int:
     return reloc.fields['r_offset'] - func.offset_in_section + 4
 
 
-def add_sign_int32(value: int) -> int:
-    """Convert a 32-bit unsigned integer to a signed integer."""
-    return value - 0x100000000 if value > 0x7FFFFFFF else value
-
-
 class stencil_database():
     """A class for loading and querying a stencil database from an ELF object file
 
@@ -231,7 +226,6 @@ class stencil_database():
                 cache.append(reloc_entry)
                 yield reloc_entry
 
-
     def get_patch(self, relocation: relocation_entry, symbol_address: int, function_offset: int, symbol_type: int) -> patch_entry:
         """Return patch positions for a provided symbol (function or object)
 
@@ -257,14 +251,12 @@ class stencil_database():
 
         if pr.type.endswith('64_PC32') or pr.type.endswith('64_PLT32'):
             # S + A - P
-            addend = add_sign_int32(pr.fields['r_addend'])
+            patch_value = symbol_address + pr.fields['r_addend'] - patch_offset
-            patch_value = symbol_address + addend - patch_offset
             #print(f" *> {pr.type} {patch_value=} {symbol_address=} {pr.fields['r_addend']=} {pr.bits=}, {function_offset=} {patch_offset=}")
 
         elif pr.type == 'R_386_PC32':
             # S + A - P
-            addend = add_sign_int32(pr.fields['r_addend'])
+            patch_value = symbol_address + pr.fields['r_addend'] - patch_offset
-            patch_value = symbol_address + addend - patch_offset
             #print(f" *> {pr.type}     {pr.symbol.name} {patch_value=} {symbol_address=} {pr.fields['r_addend']=} {bin(pr.fields['r_addend'])} {pr.bits=}, {function_offset=} {patch_offset=}")
 
         elif pr.type == 'R_386_32':

tests/test_ops_armv7mthumb.py

@@ -1,174 +0,0 @@
-from copapy import NumLike, iif, value
-from copapy.backend import Store, compile_to_dag, add_read_value_remote
-import subprocess
-from copapy import _binwrite
-import copapy.backend as backend
-import os
-import warnings
-import re
-import struct
-import pytest
-import copapy as cp
-
-if os.name == 'nt':
-    # On Windows wsl and qemu-user is required:
-    # sudo apt install qemu-user
-    qemu_command = ['wsl', 'qemu-arm']
-else:
-    qemu_command = ['qemu-arm']
-
-
-def parse_results(log_text: str) -> dict[int, bytes]:
-    regex = r"^READ_DATA offs=(\d*) size=(\d*) data=(.*)$"
-    matches = re.finditer(regex, log_text, re.MULTILINE)
-    var_dict: dict[int, bytes] = {}
-
-    for match in matches:
-        value_str: list[str] = match.group(3).strip().split(' ')
-        #print('--', value_str)
-        value = bytes(int(v, base=16) for v in value_str)
-        if len(value) <= 8:
-            var_dict[int(match.group(1))] = value
-
-    return var_dict
-
-
-def run_command(command: list[str]) -> str:
-    result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8', check=False)
-    assert result.returncode != 11, f"SIGSEGV (segmentation fault)\n -Error occurred: {result.stderr}\n -Output: {result.stdout}"
-    assert result.returncode == 0, f"\n -Error occurred: {result.stderr}\n -Output: {result.stdout}"
-    return result.stdout
-
-
-def check_for_qemu() -> bool:
-    command = qemu_command + ['--version']
-    try:
-        result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
-    except Exception:
-        return False
-    return result.returncode == 0
-
-
-def function1(c1: NumLike) -> list[NumLike]:
-    return [c1 / 4, c1 / -4, c1 // 4, c1 // -4, (c1 * -1) // 4,
-            c1 * 4, c1 * -4,
-            c1 + 4, c1 - 4,
-            c1 > 2, c1 > 100, c1 < 4, c1 < 100]
-
-def function1ex(c1: NumLike) -> list[NumLike]:
-    return [c1 // 4]
-
-
-def function2(c1: NumLike) -> list[NumLike]:
-    return [c1 * 4.44, c1 * -4.44]
-
-
-def function3(c1: NumLike) -> list[NumLike]:
-    return [c1 / 4]
-
-
-def function4(c1: NumLike) -> list[NumLike]:
-    return [c1 == 9, c1 == 4, c1 != 9, c1 != 4]
-
-
-def function5(c1: NumLike) -> list[NumLike]:
-    return [c1 == True, c1 == False, c1 != True, c1 != False, c1 / 2, c1 + 2]
-
-
-def function6(c1: NumLike) -> list[NumLike]:
-    return [c1 == True]
-
-
-def iiftests(c1: NumLike) -> list[NumLike]:
-    return [iif(c1 > 5, 8, 9),
-            iif(c1 < 5, 8.5, 9.5),
-            iif(1 > 5, 3.3, 8.8) + c1,
-            iif(1 < 5, c1 * 3.3, 8.8),
-            iif(c1 < 5, c1 * 3.3, 8.8)]
-
-
-@pytest.mark.runner
-def test_compile():
-    c_i = value(9)
-    c_f = value(1.111)
-    c_b = value(True)
-
-    ret_test = function1(c_i) + function1(c_f) + function2(c_i) + function2(c_f) + function3(c_i) + function4(c_i) + function5(c_b) + [value(9) % 2] + iiftests(c_i) + iiftests(c_f) + [cp.asin(c_i/10)]
-    ret_ref = function1(9) + function1(1.111) + function2(9) + function2(1.111) + function3(9) + function4(9) + function5(True) + [9 % 2] + iiftests(9) + iiftests(1.111) + [cp.asin(9/10)]
-
-    out = [Store(r) for r in ret_test]
-
-    sdb = backend.stencil_db_from_package('armv7mthumb')
-    dw, variables = compile_to_dag(out, sdb)
-
-    #dw.write_com(_binwrite.Command.READ_DATA)
-    #dw.write_int(0)
-    #dw.write_int(28)
-
-    du = dw.copy()
-    dw.write_com(_binwrite.Command.RUN_PROG)
-    du.write_com(_binwrite.Command.DUMP_CODE)
-
-    for v in ret_test:
-        assert isinstance(v, value)
-        add_read_value_remote(dw, variables, v.net)
-
-    #dw.write_com(_binwrite.Command.READ_DATA)
-    #dw.write_int(0)
-    #dw.write_int(28)
-
-    dw.write_com(_binwrite.Command.END_COM)
-    du.write_com(_binwrite.Command.END_COM)
-
-    #print('* Data to runner:')
-    #dw.print()
-
-    dw.to_file('build/runner/test-armv7mthumb.copapy')
-    du.to_file('build/runner/test-armv7mthumb-dump.copapy')
-
-    if not check_for_qemu():
-        warnings.warn("qemu-armv7 not found, armv7 test skipped!", UserWarning)
-        return
-    if not os.path.isfile('build/runner/coparun-armv7'):
-        warnings.warn("armv7 runner not found, armv7 test skipped!", UserWarning)
-        return
-
-    print('----- Dump code...')
-    command = qemu_command + ['build/runner/coparun-armv7', 'build/runner/test-armv7mthumb-dump.copapy', 'build/runner/test.copapy-armv7mthumb.bin']
-    result = run_command(command)
-
-    print('----- Run code...')
-    command = qemu_command + ['build/runner/coparun-armv7', 'build/runner/test-armv7mthumb.copapy']
-    result = run_command(command)
-
-
-    print('* Output from runner:\n--')
-    print(result)
-    print('--')
-
-    assert 'Return value: 1' in result
-
-    result_data = parse_results(result)
-
-    for test, ref in zip(ret_test, ret_ref):
-        assert isinstance(test, value)
-        address = variables[test.net][0]
-        data = result_data[address]
-        if test.dtype == 'int':
-            val = int.from_bytes(data, sdb.byteorder, signed=True)
-        elif test.dtype == 'bool':
-            val = bool.from_bytes(data, sdb.byteorder)
-        elif test.dtype == 'float':
-            en = {'little': '<', 'big': '>'}[sdb.byteorder]
-            val = struct.unpack(en + 'f', data)[0]
-            assert isinstance(val, float)
-        else:
-            raise Exception(f"Unknown type: {test.dtype}")
-        print('+', val, ref, test.dtype, f"  addr={address}")
-        for t in (int, float, bool):
-            assert isinstance(val, t) == isinstance(ref, t), f"Result type does not match for {val} and {ref}"
-            assert val == pytest.approx(ref, 1e-5), f"Result does not match: {val} and reference: {ref}"  # pyright: ignore[reportUnknownMemberType]
-
-
-if __name__ == "__main__":
-    test_compile()

tests/test_ops_armv7thumb.py

@@ -98,7 +98,7 @@ def test_compile():
 
     out = [Store(r) for r in ret_test]
 
-    sdb = backend.stencil_db_from_package('armv7thumb')
+    sdb = backend.stencil_db_from_package('armv7mthumb')
     dw, variables = compile_to_dag(out, sdb)
 
     #dw.write_com(_binwrite.Command.READ_DATA)
@@ -129,16 +129,16 @@ def test_compile():
     if not check_for_qemu():
         warnings.warn("qemu-armv7 not found, armv7 test skipped!", UserWarning)
         return
-    if not os.path.isfile('build/runner/coparun-armv7'):
+    if not os.path.isfile('build/runner/coparun-armv7thumb'):
-        warnings.warn("armv7 runner not found, armv7 test skipped!", UserWarning)
+        warnings.warn("armv7thumb runner not found, armv7thumb test skipped!", UserWarning)
         return
 
     print('----- Dump code...')
-    command = qemu_command + ['build/runner/coparun-armv7', 'build/runner/test-armv7thumb-dump.copapy', 'build/runner/test.copapy-armv7thumb.bin']
+    command = qemu_command + ['build/runner/coparun-armv7thumb', 'build/runner/test-armv7thumb-dump.copapy', 'build/runner/test.copapy-armv7thumb.bin']
     result = run_command(command)
 
     print('----- Run code...')
-    command = qemu_command + ['build/runner/coparun-armv7', 'build/runner/test-armv7thumb.copapy']
+    command = qemu_command + ['build/runner/coparun-armv7thumb', 'build/runner/test-armv7thumb.copapy']
     result = run_command(command)
 
 

tests/test_stencil_db.py

@@ -19,18 +19,11 @@ def test_start_end_function():
 
         if symbol.relocations and symbol.relocations[-1].symbol.info == 'STT_NOTYPE':
 
-            if symbol.section and symbol.section.name == '.text':
+            print('-', sym_name, get_stencil_position(symbol), len(symbol.data))
-                print('SKIP', sym_name, '(Aux function, not a stencil)')
-                continue
-
-            if symbol.section:
-                function_size = symbol.section.fields['sh_size']  # len(symbol.data) excludes nop after the function
-
-                print('-', sym_name, get_stencil_position(symbol), function_size)
 
             start, end = get_stencil_position(symbol)
 
-                assert (start >= 0 and end >= start and end <= function_size)
+            assert start >= 0 and end >= start and end <= len(symbol.data)
 
 
 def test_aux_functions():

tools/build.sh

@@ -128,7 +128,6 @@ if [[ "$ARCH" == "arm-v7" || "$ARCH" == "all" ]]; then
         $DEST/stencils_armv7_O3.o \
         > build/stencils/stencils_armv7_O3.asm
 
-    # The same runner for all ARM7
     arm-linux-gnueabihf-gcc \
         -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -marm -static \
         -Wall -Wextra -Wconversion -Wsign-conversion \
@@ -162,9 +161,8 @@ if [[ "$ARCH" == "arm-v7-thumb" || "$ARCH" == "all" ]]; then
         $DEST/stencils_armv7thumb_O3.o \
         > build/stencils/stencils_armv7thumb_O3.asm
 
-    # The same runner for all ARM7
     arm-linux-gnueabihf-gcc \
-        -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -static \
+        -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb -static \
         -Wall -Wextra -Wconversion -Wsign-conversion \
         -Wshadow -Wstrict-overflow -O3 \
         -DENABLE_LOGGING \
@@ -196,9 +194,8 @@ if [[ "$ARCH" == "arm-v7m-thumb" || "$ARCH" == "all" ]]; then
         $DEST/stencils_armv7mthumb_O3.o \
         > build/stencils/stencils_armv7mthumb_O3.asm
 
-    # The same runner for all ARM7
     arm-linux-gnueabihf-gcc \
-        -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -static \
+        -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb -static \
         -Wall -Wextra -Wconversion -Wsign-conversion \
         -Wshadow -Wstrict-overflow -O3 \
         -DENABLE_LOGGING \

tools/crosscompile.sh

@@ -43,12 +43,12 @@ arm-none-eabi-ld -r $STMP /object_files/musl_objects_armv7.o $LIBGCC -o $DEST/st
 
 # ARMv7 Thumb for Cortex-A with hardware fp
 arm-none-eabi-gcc -march=armv7-a -mfpu=neon-vfpv3 -mfloat-abi=hard -mthumb $FLAGS -$OPT -c $SRC -o $STMP
-LIBGCC=$(arm-none-eabi-gcc -march=armv7 -mfpu=vfp3 -mthumb -print-libgcc-file-name)
+LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name)
 arm-none-eabi-ld -r $STMP /object_files/musl_objects_armv7thumb.o $LIBGCC -o $DEST/stencils_armv7thumb_$OPT.o
 
 # Armv7 Thumb for Cortex-M3..7 hardware fp
 arm-none-eabi-gcc -march=armv7e-m -mfpu=fpv4-sp-d16 -mfloat-abi=hard -mthumb $FLAGS -$OPT -c $SRC -o $STMP
-LIBGCC=$(arm-none-eabi-gcc -march=armv7e-m -mfpu=fpv4-sp-d16 -mfloat-abi=hard -mthumb -print-libgcc-file-name)
+LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name)
 arm-none-eabi-ld -r $STMP /object_files/musl_objects_armv7mthumb.o $LIBGCC -o $DEST/stencils_armv7mthumb_$OPT.o
 
 # PowerPC64LE