diff --git a/.github/workflows/build_docker_image.yml b/.github/workflows/build_docker_image.yml index b67c98e..a236d7d 100644 --- a/.github/workflows/build_docker_image.yml +++ b/.github/workflows/build_docker_image.yml @@ -63,6 +63,37 @@ jobs: - name: Build & Push Docker image run: docker buildx build --platform linux/arm64 --push -t $IMAGE_NAME tools/qemu_test/ + docker-build-armv6: + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set image name + run: echo "IMAGE_NAME=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/armv6_test:1" >> $GITHUB_ENV + + - name: Build & Push Docker image + run: docker buildx build --platform linux/arm/v6 --push -t $IMAGE_NAME tools/qemu_test/ + + docker-build-armv7: runs-on: ubuntu-latest env: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f60e2e..3ec0d72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,6 +153,35 @@ jobs: name: runner-linux-arm64 path: build/runner/* + build-armv6: + needs: [build_stencils] + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: stencil-object-files + path: src/copapy/obj + - name: Set up QEMU for ARMv6 + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/arm/v6 + - name: Use ARMv6 container + run: | + docker run --rm -v $PWD:/app -w /app --platform linux/arm/v6 ghcr.io/nonannet/armv6_test:1 \ + bash -lc "pip install . && \ + mkdir -p build/runner && \ + gcc -O3 -DENABLE_LOGGING -o build/runner/coparun src/coparun/runmem.c \ + src/coparun/coparun.c src/coparun/mem_man.c && \ + pytest && \ + bash tools/create_asm.sh" + + - uses: actions/upload-artifact@v4 + with: + name: runner-linux-armv6 + path: build/runner/* + build-armv7: needs: [build_stencils] runs-on: ubuntu-latest @@ -231,7 +260,7 @@ jobs: path: build/runner/* release-stencils: - needs: [build_stencils, build-ubuntu, build-windows, build-arm64, build-armv7] + needs: [build_stencils, build-ubuntu, build-windows, build-arm64, build-armv6, build-armv7] runs-on: ubuntu-latest if: github.ref == 'refs/heads/main' && github.event_name == 'push' permissions: @@ -263,6 +292,7 @@ jobs: cp tmp/musl-object-files/* release/ cp tmp/runner-linux/coparun release/ cp tmp/runner-linux-arm64/coparun release/coparun-aarch64 + cp tmp/runner-linux-armv6/coparun release/coparun-armv6 cp tmp/runner-linux-armv7/coparun release/coparun-armv7 cp tmp/runner-win/coparun*.exe release/ diff --git a/src/copapy/_basic_types.py b/src/copapy/_basic_types.py index 70f23f0..9ed7f49 100644 --- a/src/copapy/_basic_types.py +++ b/src/copapy/_basic_types.py @@ -64,6 +64,9 @@ class Node: def __hash__(self) -> int: return self.node_hash + def __eq__(self, value: object) -> bool: + return isinstance(value, Node) and self.node_hash == value.node_hash # TODO: change to 64 bit hash + class Net: """A Net represents a scalar type in the computation graph - or more generally it @@ -79,7 +82,7 @@ class Net: def __repr__(self) -> str: names = get_var_name(self) - return f"{'name:' + names[0] if names else 'id:' + str(hash(self))[-5:]}" + return f"{'name:' + names[0] if names else 'h:' + str(hash(self))[-5:]}" def __hash__(self) -> int: return self.source.node_hash @@ -352,7 +355,7 @@ class CPConstant(Node): self.name = 'const_' + self.dtype self.args = tuple() - self.node_hash = hash(value) ^ hash(self.dtype) if anonymous else id(self) + self.node_hash = ((value if isinstance(value, int) else hash(value)) + 0x1_0000_0000) ^ hash(self.dtype) if anonymous else id(self) # TODO: Simplify hash and compare class Write(Node): diff --git a/src/copapy/_compiler.py b/src/copapy/_compiler.py index 6af41b2..16676c6 100644 --- a/src/copapy/_compiler.py +++ b/src/copapy/_compiler.py @@ -380,7 +380,7 @@ def compile_to_dag(node_list: Iterable[Node], sdb: stencil_database) -> tuple[bi dw.write_int(start) dw.write_int(lengths) dw.write_value(net.source.value, lengths) - print(f'+ {net.dtype} {net.source.value}') + #print(f'+ {net.dtype} {net.source.value}') # prep auxiliary_functions code_section_layout, func_addr_lookup, aux_func_len = get_aux_func_layout(aux_function_names, sdb) diff --git a/src/copapy/_stencils.py b/src/copapy/_stencils.py index a5b16e3..b7864c0 100644 --- a/src/copapy/_stencils.py +++ b/src/copapy/_stencils.py @@ -95,11 +95,14 @@ def get_last_call_in_function(func: pelfy.elf_symbol) -> int: # Find last relocation in function assert func.relocations, f'No call function in stencil function {func.name}.' reloc = func.relocations[-1] - # Assume the call instruction is 4 bytes long for relocations with less than 32 bit and 5 bytes otherwise - instruction_lengths = 4 if reloc.bits < 32 else 5 - address_field_length = 4 - #print(f"-> {[r.fields['r_offset'] - func.fields['st_value'] for r in func.relocations]}") - return reloc.fields['r_offset'] - func.fields['st_value'] + address_field_length - instruction_lengths + if reloc.symbol.name.startswith('dummy_'): + return -0xFFFF # Last relocation is not a jump + else: + # Assume the call instruction is 4 bytes long for relocations with less than 32 bit and 5 bytes otherwise + instruction_lengths = 4 if reloc.bits < 32 else 5 + address_field_length = 4 + #print(f"-> {[r.fields['r_offset'] - func.fields['st_value'] for r in func.relocations]}") + return reloc.fields['r_offset'] - func.fields['st_value'] + address_field_length - instruction_lengths def get_op_after_last_call_in_function(func: pelfy.elf_symbol) -> int: @@ -305,6 +308,12 @@ class stencil_database(): symbol_type = symbol_type + 0x04 # Absolut value scale = 0x10000 + elif pr.type.endswith('_ABS32'): + # R_ARM_ABS32 + # S + A (replaces full 32 bit) + patch_value = symbol_address + pr.fields['r_addend'] + symbol_type = symbol_type + 0x03 # Relative to data section + else: raise NotImplementedError(f"Relocation type {pr.type} in {relocation.pelfy_reloc.target_section.name} pointing to {relocation.pelfy_reloc.symbol.name} not implemented") diff --git a/tests/test_jit_decorator.py b/tests/test_jit_decorator.py index 8e43449..2f9f051 100644 --- a/tests/test_jit_decorator.py +++ b/tests/test_jit_decorator.py @@ -48,6 +48,20 @@ def slow_31bit_int_list_hash(data: list[int], rounds: int = 5)-> int: return state +def test_hash_without_decorator(): + nums = [12, 99, 2024] + h_ref = slow_31bit_int_list_hash(nums) + h = slow_31bit_int_list_hash([cp.value(num) for num in nums]) + + tg = cp.Target() + tg.compile(h) + tg.run() + + assert isinstance(h, cp.value) + assert tg.read_value(h) == h_ref + print(tg.read_value(h), h_ref) + + def test_decorator(): sumv = 0 y = 5.7 @@ -57,11 +71,10 @@ def test_decorator(): assert abs(sumv - 166542418649.28778) < 1e14, sumv + def test_hash(): nums = [12, 99, 2024] h_ref = slow_31bit_int_list_hash(nums) - print(h_ref) - h = cp.jit(slow_31bit_int_list_hash)(nums) - print(h) + print(h, h_ref) assert h == h_ref diff --git a/tests/test_ops_armv6.py b/tests/test_ops_armv6.py new file mode 100644 index 0000000..1797c95 --- /dev/null +++ b/tests/test_ops_armv6.py @@ -0,0 +1,171 @@ +from copapy import NumLike, iif, value +from copapy.backend import Write, compile_to_dag, add_read_command +import subprocess +from copapy import _binwrite +import copapy.backend as backend +import os +import warnings +import re +import struct +import pytest +import copapy as cp + +if os.name == 'nt': + # On Windows wsl and qemu-user is required: + # sudo apt install qemu-user + qemu_command = ['wsl', 'qemu-arm'] +else: + qemu_command = ['qemu-arm'] + + +def parse_results(log_text: str) -> dict[int, bytes]: + regex = r"^READ_DATA offs=(\d*) size=(\d*) data=(.*)$" + matches = re.finditer(regex, log_text, re.MULTILINE) + var_dict: dict[int, bytes] = {} + + for match in matches: + value_str: list[str] = match.group(3).strip().split(' ') + #print('--', value_str) + value = bytes(int(v, base=16) for v in value_str) + if len(value) <= 8: + var_dict[int(match.group(1))] = value + + return var_dict + + +def run_command(command: list[str]) -> str: + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8', check=False) + assert result.returncode != 11, f"SIGSEGV (segmentation fault)\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + assert result.returncode == 0, f"\n -Error occurred: {result.stderr}\n -Output: {result.stdout}" + return result.stdout + + +def check_for_qemu() -> bool: + command = qemu_command + ['--version'] + try: + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + except Exception: + return False + return result.returncode == 0 + + +def function1(c1: NumLike) -> list[NumLike]: + return [c1 / 4, c1 / -4, c1 // 4, c1 // -4, (c1 * -1) // 4, + c1 * 4, c1 * -4, + c1 + 4, c1 - 4, + c1 > 2, c1 > 100, c1 < 4, c1 < 100] + + +def function2(c1: NumLike) -> list[NumLike]: + return [c1 * 4.44, c1 * -4.44] + + +def function3(c1: NumLike) -> list[NumLike]: + return [c1 / 4] + + +def function4(c1: NumLike) -> list[NumLike]: + return [c1 == 9, c1 == 4, c1 != 9, c1 != 4] + + +def function5(c1: NumLike) -> list[NumLike]: + return [c1 == True, c1 == False, c1 != True, c1 != False, c1 / 2, c1 + 2] + + +def function6(c1: NumLike) -> list[NumLike]: + return [c1 == True] + + +def iiftests(c1: NumLike) -> list[NumLike]: + return [iif(c1 > 5, 8, 9), + iif(c1 < 5, 8.5, 9.5), + iif(1 > 5, 3.3, 8.8) + c1, + iif(1 < 5, c1 * 3.3, 8.8), + iif(c1 < 5, c1 * 3.3, 8.8)] + + +@pytest.mark.runner +def test_compile(): + c_i = value(9) + c_f = value(1.111) + c_b = value(True) + + ret_test = function1(c_i) + function1(c_f) + function2(c_i) + function2(c_f) + function3(c_i) + function4(c_i) + function5(c_b) + [value(9) % 2] + iiftests(c_i) + iiftests(c_f) + [cp.asin(c_i/10)] + ret_ref = function1(9) + function1(1.111) + function2(9) + function2(1.111) + function3(9) + function4(9) + function5(True) + [9 % 2] + iiftests(9) + iiftests(1.111) + [cp.asin(9/10)] + + #ret_test = (c_i * 100 // 5, c_f * 10 // 5) + #ret_ref = (9 * 100 // 5, 1.111 * 10 // 5) + + out = [Write(r) for r in ret_test] + + sdb = backend.stencil_db_from_package('armv6') + dw, variables = compile_to_dag(out, sdb) + + #dw.write_com(_binwrite.Command.READ_DATA) + #dw.write_int(0) + #dw.write_int(28) + + # run program command + dw.write_com(_binwrite.Command.RUN_PROG) + #dw.write_com(_binwrite.Command.DUMP_CODE) + + for v in ret_test: + assert isinstance(v, value) + add_read_command(dw, variables, v.net) + + #dw.write_com(_binwrite.Command.READ_DATA) + #dw.write_int(0) + #dw.write_int(28) + + dw.write_com(_binwrite.Command.END_COM) + + #print('* Data to runner:') + #dw.print() + + dw.to_file('build/runner/test-armv6.copapy') + + if not check_for_qemu(): + warnings.warn("qemu-armv6 not found, armv6 test skipped!", UserWarning) + return + if not os.path.isfile('build/runner/coparun-armv6'): + warnings.warn("armv6 runner not found, armv6 test skipped!", UserWarning) + return + + command = qemu_command + ['build/runner/coparun-armv6', 'build/runner/test-armv6.copapy'] + ['build/runner/test-armv6.copapy.bin'] + #try: + result = run_command(command) + #except FileNotFoundError: + # warnings.warn(f"Test skipped, executable not found.", UserWarning) + # return + + #print('* Output from runner:\n--') + #print(result) + #print('--') + + assert 'Return value: 1' in result + + result_data = parse_results(result) + + for test, ref in zip(ret_test, ret_ref): + assert isinstance(test, value) + address = variables[test.net][0] + data = result_data[address] + if test.dtype == 'int': + val = int.from_bytes(data, sdb.byteorder, signed=True) + elif test.dtype == 'bool': + val = bool.from_bytes(data, sdb.byteorder) + elif test.dtype == 'float': + en = {'little': '<', 'big': '>'}[sdb.byteorder] + val = struct.unpack(en + 'f', data)[0] + assert isinstance(val, float) + else: + raise Exception(f"Unknown type: {test.dtype}") + print('+', val, ref, test.dtype, f" addr={address}") + for t in (int, float, bool): + assert isinstance(val, t) == isinstance(ref, t), f"Result type does not match for {val} and {ref}" + assert val == pytest.approx(ref, 1e-5), f"Result does not match: {val} and reference: {ref}" # pyright: ignore[reportUnknownMemberType] + + +if __name__ == "__main__": + #test_compile() + test_slow_31bit_int_list_hash() diff --git a/tests/test_ops_armv7.py b/tests/test_ops_armv7.py index 415076f..31e8e05 100644 --- a/tests/test_ops_armv7.py +++ b/tests/test_ops_armv7.py @@ -168,4 +168,4 @@ def test_compile(): if __name__ == "__main__": #test_example() - test_compile() + test_slow_31bit_int_list_hash() diff --git a/tests/test_ops_x86.py b/tests/test_ops_x86.py index 13c902e..c54d663 100644 --- a/tests/test_ops_x86.py +++ b/tests/test_ops_x86.py @@ -156,7 +156,7 @@ def test_compile(): for test, ref in zip(ret_test, ret_ref): assert isinstance(test, value) - address = variables[test][0] + address = variables[test.net][0] data = result_data[address] if test.dtype == 'int': val = int.from_bytes(data, sdb.byteorder, signed=True) diff --git a/tools/build.sh b/tools/build.sh index ae78afc..2ece1e8 100644 --- a/tools/build.sh +++ b/tools/build.sh @@ -21,6 +21,17 @@ gcc -Wall -Wextra -Wconversion -Wsign-conversion \ src/coparun/runmem.c src/coparun/coparun.c src/coparun/mem_man.c -o build/runner/coparun +echo "--------------arm-v6 32 bit----------------" +LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) +#LIBM=$(arm-none-eabi-gcc -print-file-name=libm.a) +#LIBC=$(arm-none-eabi-gcc -print-file-name=libc.a) + +arm-none-eabi-gcc -fno-pic -ffunction-sections -march=armv6 -mfpu=vfp -mfloat-abi=hard -marm -c $SRC -O3 -o build/stencils/stencils.o +arm-none-eabi-ld -r build/stencils/stencils.o build/musl/musl_objects_armv6.o $LIBGCC -o $DEST/stencils_armv6_O3.o +arm-none-eabi-objdump -d -x $DEST/stencils_armv6_O3.o > build/stencils/stencils_armv6_O3.asm +arm-linux-gnueabihf-gcc -march=armv6 -mfpu=vfp -mfloat-abi=hard -marm -static -Wall -Wextra -Wconversion -Wsign-conversion -Wshadow -Wstrict-overflow -O3 -DENABLE_LOGGING src/coparun/runmem.c src/coparun/coparun.c src/coparun/mem_man.c -o build/runner/coparun-armv6 + + echo "--------------arm-v7 32 bit----------------" LIBGCC=$(arm-none-eabi-gcc -print-libgcc-file-name) #LIBM=$(arm-none-eabi-gcc -print-file-name=libm.a)