nicolas
/
copapy
mirror of https://github.com/Nonannet/copapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

build ci added

This commit is contained in:
Nicolas 2025-10-04 22:57:45 +02:00
parent e8bf9f1a26
commit a19fd88993
19 changed files with 326 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
.github/workflows/build_wheels.yml vendored Normal file
View File

@ -0,0 +1,87 @@
name: Build and Publish Wheels
on:
push:
tags:
- "v*"
workflow_dispatch:
jobs:
build_stencils:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- name: Install cross compilers
run: |
sudo apt-get update
sudo apt-get install -y \
gcc-12-aarch64-linux-gnu \
gcc-12-arm-linux-gnueabihf \
gcc-12-mips-linux-gnu \
gcc-12-riscv64-linux-gnu
- name: Build object files
run: bash src/copapy/obj/crosscompile.sh
- uses: actions/upload-artifact@v3
with:
name: stencil-object-files
path: src/copapy/obj/*.o
build_wheels:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-20.04, windows-latest, macos-latest]
steps:
- uses: actions/checkout@v3
- uses: actions/download-artifact@v3
with:
name: stencil-object-files
path: src/copapy/obj
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.x"
# Only needed for Linux ARM builds
- name: Set up QEMU
if: runner.os == 'Linux'
uses: docker/setup-qemu-action@v2
with:
platforms: all
- name: Install dependencies
run: python -m pip install --upgrade pip cibuildwheel cython setuptools wheel
- name: Build wheels
run: cibuildwheel --output-dir wheelhouse
env:
# Multi-arch builds
CIBW_ARCHS_LINUX: "x86_64 aarch64 armv7" # ppc64le s390x
CIBW_ARCHS_MACOS: "universal2" # x86_64 arm64
CIBW_ARCHS_WINDOWS: "AMD64 x86"
- uses: actions/upload-artifact@v3
with:
name: wheels
path: wheelhouse/*.whl
# publish:
# needs: [build_wheels]
# runs-on: ubuntu-latest
# steps:
# - uses: actions/download-artifact@v3
# with:
# name: wheels
# path: wheelhouse
# - name: Publish to PyPI
# uses: pypa/gh-action-pypi-publish@v1.8.6
# with:
# user: __token__
# password: ${{ secrets.PYPI_API_TOKEN }}

12
.github/workflows/ci.yml vendored
View File

@ -35,15 +35,15 @@ jobs:
- name: Build ops obj files and runner
run: bash build.sh
#- name: Lint code with flake8
# run: flake8
#- name: Type checking with mypy
# run: mypy
- name: Run tests with pytest
run: pytest
- name: Type checking with mypy
run: mypy
#- name: Lint code with flake8
# run: flake8
- name: Upload obj files
uses: actions/upload-artifact@v4
if: strategy.job-index == 0

14
build.sh
View File

@ -2,12 +2,14 @@
set -e
set -v
python src/copapy/generate_stencils.py
mkdir -p src/copapy/obj
gcc -c src/copapy/stencils.c -o src/copapy/obj/stencils_x86_64.o
gcc -c src/copapy/stencils.c -O1 -o src/copapy/obj/stencils_x86_64_O1.o
gcc -c src/copapy/stencils.c -O2 -o src/copapy/obj/stencils_x86_64_O2.o
gcc -c src/copapy/stencils.c -O3 -o src/copapy/obj/stencils_x86_64_O3.o
SRC=src/copapy/stencils.c
DEST=src/copapy/obj
mkdir -p $DEST
gcc -c $SRC -o $DEST/stencils_x86_64.o
gcc -c $SRC -O1 -o $DEST/stencils_x86_64_O1.o
gcc -c $SRC -O2 -o $DEST/stencils_x86_64_O2.o
gcc -c $SRC -O3 -o $DEST/stencils_x86_64_O3.o
mkdir bin -p
gcc -Wall -Wextra -Wconversion -Wsign-conversion -Wshadow -Wstrict-overflow -Werror -g src/coparun/runmem.c src/coparun/coparun.c -o bin/coparun
gcc -Wall -Wextra -Wconversion -Wsign-conversion -Wshadow -Wstrict-overflow -Werror -g src/coparun/runmem.c src/coparun/coparun.c src/coparun/mem_man.c -o bin/coparun
#x86_64-w64-mingw32-gcc -Wall -Wextra -Wconversion -Wsign-conversion -Wshadow -Wstrict-overflow -Werror src/runner/runmem2.c -Wall -O3 -o bin/runmem2.exe

2
pyproject.toml
View File

@ -28,7 +28,7 @@ build-backend = "setuptools.build_meta"
where = ["src"]
[tool.setuptools.package-data]
copapy = ["*.o"]
copapy = ["obj/*.o", "py.typed"]
[project.optional-dependencies]
dev = [

42
src/copapy/__init__.py
View File

@ -1,22 +1,21 @@
# import pkgutil
import pkgutil
from typing import Generator, Iterable, Any
from . import binwrite as binw
from .stencil_db import stencil_database
from collections import defaultdict, deque
from coparun_module import coparun, read_data_mem
import struct
import platform
Operand = type['Net'] | float | int
def get_var_name(var: Any, scope: dict[str, Any] = globals()) -> list[str]:
return [name for name, value in scope.items() if value is var]
# _ccode = pkgutil.get_data(__name__, 'stencils.c')
# assert _ccode is not None
sdb = stencil_database('src/copapy/obj/stencils_x86_64_O3.o')
_arch = platform.machine()
_stencil_data = pkgutil.get_data(__name__, f"obj/stencils_{_arch}_O3.o")
assert _stencil_data
generic_sdb = stencil_database(_stencil_data)
class Node:
@ -71,7 +70,7 @@ class Net:
def __lt__(self, other: Any) -> 'Net':
return _add_op('gt', [other, self])
def __eq__(self, other: Any) -> 'Net':
def __eq__(self, other: Any) -> 'Net': # type: ignore
return _add_op('eq', [self, other])
def __mod__(self, other: Any) -> 'Net':
@ -123,10 +122,10 @@ def _add_op(op: str, args: list[Any], commutative: bool = False) -> Net:
typed_op = '_'.join([op] + [a.dtype for a in arg_nets])
if typed_op not in sdb.function_definitions:
if typed_op not in generic_sdb.function_definitions:
raise ValueError(f"Unsupported operand type(s) for {op}: {' and '.join([a.dtype for a in arg_nets])}")
result_type = sdb.function_definitions[typed_op].split('_')[0]
result_type = generic_sdb.function_definitions[typed_op].split('_')[0]
result_net = Net(result_type, Op(typed_op, arg_nets))
@ -301,7 +300,7 @@ def add_write_ops(net_node_list: list[tuple[Net | None, Node]], const_nets: list
else:
yield None, node
if net in read_back_nets and net not in stored_nets:
if net and net in read_back_nets and net not in stored_nets:
yield net, Write(net)
stored_nets.add(net)
@ -414,8 +413,12 @@ def compile_to_instruction_list(node_list: Iterable[Node], sdb: stencil_database
class Target():
def __init__(self, arch: str = 'x86_64', optimization: str = 'O3') -> None:
self.sdb = stencil_database(f"src/copapy/obj/stencils_{arch}_{optimization}.o")
def __init__(self, arch: str = 'native', optimization: str = 'O3') -> None:
if arch == 'native':
arch = platform.machine()
stencil_data = pkgutil.get_data(__name__, f"obj/stencils_{arch}_{optimization}.o")
assert stencil_data
self.sdb = stencil_database(stencil_data)
self._variables: dict[Net, tuple[int, int, str]] = dict()
@ -438,7 +441,7 @@ class Target():
def run(self) -> None:
# set entry point and run code
dw = binw.data_writer(self.sdb.byteorder)
dw.write_com(binw.Command.SET_ENTR_POINT)
dw.write_com(binw.Command.RUN_PROG)
dw.write_int(0)
dw.write_com(binw.Command.END_PROG)
assert coparun(dw.get_data()) > 0
@ -471,10 +474,11 @@ class Target():
else:
raise ValueError(f"Unsupported variable type: {var_type}")
def read_variable_remote(self, bw: binw.data_writer, net: Net) -> None:
def read_variable_remote(self, net: Net) -> None:
assert net in self._variables, f"Variable {net} not found in data writer variables"
dw = binw.data_writer(self.sdb.byteorder)
addr, lengths, _ = self._variables[net]
bw.write_com(binw.Command.READ_DATA)
bw.write_int(addr)
bw.write_int(lengths)
dw = binw.data_writer(self.sdb.byteorder)
dw.write_com(binw.Command.READ_DATA)
dw.write_int(addr)
dw.write_int(lengths)
assert coparun(dw.get_data()) > 0

4
src/copapy/binwrite.py
View File

@ -1,12 +1,12 @@
from enum import Enum
from typing import Literal, Any
from typing import Literal
import struct
Command = Enum('Command', [('ALLOCATE_DATA', 1), ('COPY_DATA', 2),
('ALLOCATE_CODE', 3), ('COPY_CODE', 4),
('PATCH_FUNC', 5), ('PATCH_OBJECT', 6),
('SET_ENTR_POINT', 64), ('READ_DATA', 65),
('RUN_PROG', 64), ('READ_DATA', 65),
('END_PROG', 256), ('FREE_MEMORY', 257)])

42
src/copapy/obj/crosscompile.sh Normal file
View File

@ -0,0 +1,42 @@
#!/bin/bash
#Setup:
#sudo apt-get update
#sudo apt-get install -y \
#gcc-12-aarch64-linux-gnu \
#gcc-12-arm-linux-gnueabihf \
#gcc-12-powerpc64le-linux-gnu \
#gcc-12-s390x-linux-gnu \
#gcc-12-mips-linux-gnu \
#gcc-12-riscv64-linux-gnu
set -e
set -v
python src/copapy/generate_stencils.py
SRC=src/copapy/stencils.c
DEST=src/copapy/obj
mkdir -p $DEST
# Native x86_64
gcc-12 -c $SRC -o $DEST/stencils_x86_64.o
# ARM64
aarch64-linux-gnu-gcc-12 -O3 -c $SRC -o $DEST/stencils_aarch64.o
# ARMv7
arm-linux-gnueabihf-gcc-12 -O3 -c $SRC -o $DEST/stencils_armv7.o
# PowerPC64LE
# powerpc64le-linux-gnu-gcc-12 -O3 -c $SRC -o $DEST/stencils_ppc64le.o
# S390x
# s390x-linux-gnu-gcc-12 -O3 -c $SRC -o $DEST/stencils_s390x.o
# Mips
mips-linux-gnu-gcc-12 -O3 -c $SRC -o $DEST/stencils_mips.o
# RISCV 32 Bit
riscv64-linux-gnu-gcc-12 -O3 -march=rv32imac -mabi=ilp32 -c $SRC -o $DEST/stencils_riscv32.o
# RISCV 64 Bit
riscv64-linux-gnu-gcc-12 -O3 -c $SRC -o $DEST/stencils_riscv64.o

0
src/copapy/py.typed Normal file
View File

7
src/copapy/stencil_db.py
View File

@ -67,8 +67,13 @@ def get_stencil_position(data: bytes, byteorder: ByteOrder) -> tuple[int, int]:
class stencil_database():
def __init__(self, obj_file: str):
def __init__(self, obj_file: str | bytes):
"""Load the stencil database from an ELF object file
"""
if isinstance(obj_file, str):
self.elf = pelfy.open_elf_file(obj_file)
else:
self.elf = pelfy.elf_file(obj_file)
#print(self.elf.symbols)

4
src/coparun/coparun.c
View File

@ -5,6 +5,7 @@
#include <sys/stat.h>
#include <unistd.h>
#include "runmem.h"
#include "mem_man.h"
int main(int argc, char *argv[]) {
if (argc != 2) {
@ -33,8 +34,7 @@ int main(int argc, char *argv[]) {
return EXIT_FAILURE;
}
//uint8_t *file_buff = get_data_memory((uint32_t)st.st_size);
uint8_t *file_buff = (uint8_t*)malloc((size_t)st.st_size);
uint8_t *file_buff = allocate_buffer_memory((uint32_t)st.st_size);
// Read file into allocated memory
if (read(fd, file_buff, (long unsigned int)st.st_size) != st.st_size) {

96
src/coparun/mem_man.c Normal file
View File

@ -0,0 +1,96 @@
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#ifdef _WIN32
#include <windows.h>
/* Windows implementations */
uint8_t *allocate_executable_memory(uint32_t num_bytes) {
uint8_t *mem = (uint8_t*)VirtualAlloc(NULL, (SIZE_T)num_bytes,
MEM_RESERVE | MEM_COMMIT,
PAGE_READWRITE);
if (mem == NULL) {
fprintf(stderr, "VirtualAlloc failed (executable): %lu\n", GetLastError());
}
return mem;
}
uint8_t *allocate_data_memory(uint32_t num_bytes) {
/* Allocate RW memory that can later be made executable. */
uint8_t *mem = (uint8_t*)VirtualAlloc(NULL, (SIZE_T)num_bytes,
MEM_RESERVE | MEM_COMMIT,
PAGE_READWRITE);
if (mem == NULL) {
fprintf(stderr, "VirtualAlloc failed (data): %lu\n", GetLastError());
}
return mem;
}
uint8_t *allocate_buffer_memory(uint32_t num_bytes) {
return (uint8_t*)malloc((size_t)num_bytes);
}
int mark_mem_executable(uint8_t *memory, uint32_t memory_len) {
if (!memory || memory_len == 0) return 0;
DWORD oldProtect = 0;
if (!VirtualProtect((LPVOID)memory, (SIZE_T)memory_len, PAGE_EXECUTE_READ, &oldProtect)) {
fprintf(stderr, "VirtualProtect failed: %lu\n", GetLastError());
return 0;
}
return 1;
}
void deallocate_memory(uint8_t *memory, uint32_t memory_len) {
if (!memory) return;
if (memory_len) {
VirtualFree((LPVOID)memory, 0, MEM_RELEASE);
} else {
free(memory);
}
}
#else
#include <sys/mman.h>
/* POSIX implementations */
uint8_t *allocate_executable_memory(uint32_t num_bytes) {
uint8_t *mem = (uint8_t*)mmap(NULL, num_bytes,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
return mem;
}
uint8_t *allocate_data_memory(uint32_t num_bytes) {
/*
Malloc can not be used since it may return a memory region too far apart
from the executable memory yielded by mmap for relative 32 bit addressing.
uint8_t *mem = (uint8_t*)malloc(num_bytes);
*/
uint8_t *mem = (uint8_t*)mmap(NULL, num_bytes,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
return mem;
}
uint8_t *allocate_buffer_memory(uint32_t num_bytes) {
return (uint8_t*)malloc((size_t)num_bytes);
}
int mark_mem_executable(uint8_t *memory, uint32_t memory_len) {
if (mprotect(memory, memory_len, PROT_READ | PROT_EXEC) == -1) {
perror("mprotect failed");
return 0;
}
return 1;
}
void deallocate_memory(uint8_t *memory, uint32_t memory_len) {
if (memory_len) munmap(memory, memory_len);
}
#endif

10
src/coparun/mem_man.h Normal file
View File

@ -0,0 +1,10 @@
#ifndef MEM_MAN_H
#define MEM_MAN_H
uint8_t *allocate_executable_memory(uint32_t num_bytes);
uint8_t *allocate_data_memory(uint32_t num_bytes);
uint8_t *allocate_buffer_memory(uint32_t num_bytes);
int mark_mem_executable(uint8_t *memory, uint32_t memory_len);
void deallocate_memory(uint8_t *memory, uint32_t memory_len);
#endif /* MEM_MAN_H */

77
src/coparun/runmem.c
View File

@ -7,46 +7,22 @@
#include <string.h>
#include <stdint.h>
#include "runmem.h"
#include "mem_man.h"
/* Definitions for globals declared extern in runmem.h */
/* Globals declared extern in runmem.h */
uint8_t *data_memory = NULL;
uint32_t data_memory_len = 0;
uint8_t *executable_memory = NULL;
uint32_t executable_memory_len = 0;
entry_point_t entr_point = NULL;
uint8_t *get_executable_memory(uint32_t num_bytes){
// Allocate executable memory
uint8_t *mem = (uint8_t*)mmap(NULL, num_bytes,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
return mem;
}
uint8_t *get_data_memory(uint32_t num_bytes) {
uint8_t *mem = (uint8_t*)mmap(NULL, num_bytes,
PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
//uint8_t *mem = (uint8_t*)malloc(num_bytes);
return mem;
}
int mark_mem_executable(){
if (mprotect(executable_memory, executable_memory_len, PROT_READ | PROT_EXEC) == -1) {
perror("mprotect failed");
return 0;
}else{
return 1;
}
}
void patch_mem_32(uint8_t *patch_addr, int32_t value){
void patch_mem_32(uint8_t *patch_addr, int32_t value) {
int32_t *val_ptr = (int32_t*)patch_addr;
*val_ptr = value;
}
int patch(uint8_t *patch_addr, uint32_t reloc_type, int32_t value){
if (reloc_type == PATCH_RELATIVE_32){
int patch(uint8_t *patch_addr, uint32_t reloc_type, int32_t value) {
if (reloc_type == PATCH_RELATIVE_32) {
patch_mem_32(patch_addr, value);
}else{
printf("Not implemented");
@ -55,14 +31,14 @@ int patch(uint8_t *patch_addr, uint32_t reloc_type, int32_t value){
return 1;
}
void free_memory(){
if (executable_memory_len) munmap(executable_memory, executable_memory_len);
if (data_memory_len) munmap(data_memory, data_memory_len);
data_memory_len = 0;
void free_memory() {
deallocate_memory(executable_memory, executable_memory_len);
deallocate_memory(data_memory, data_memory_len);
executable_memory_len = 0;
data_memory_len = 0;
}
int parse_commands(uint8_t *bytes){
int parse_commands(uint8_t *bytes) {
int32_t value;
uint32_t command;
uint32_t reloc_type;
@ -72,18 +48,13 @@ int parse_commands(uint8_t *bytes){
int err_flag = 0;
uint32_t rel_entr_point;
while(!err_flag){
while(!err_flag) {
command = *(uint32_t*)bytes;
bytes += 4;
switch(command) {
case FREE_MEMORY:
size = *(uint32_t*)bytes; bytes += 4;
free_memory();
break;
case ALLOCATE_DATA:
size = *(uint32_t*)bytes; bytes += 4;
data_memory = get_data_memory(size);
data_memory = allocate_data_memory(size);
data_memory_len = size;
printf("ALLOCATE_DATA size=%i mem_addr=%p\n", size, (void*)data_memory);
break;
@ -97,7 +68,7 @@ int parse_commands(uint8_t *bytes){
case ALLOCATE_CODE:
size = *(uint32_t*)bytes; bytes += 4;
executable_memory = get_executable_memory(size);
executable_memory = allocate_executable_memory(size);
executable_memory_len = size;
printf("ALLOCATE_CODE size=%i mem_addr=%p\n", size, (void*)executable_memory);
//printf("# d %i c %i off %i\n", data_memory, executable_memory, data_offs);
@ -131,24 +102,18 @@ int parse_commands(uint8_t *bytes){
return EXIT_FAILURE;
}
patch(executable_memory + offs, reloc_type, value + data_offs);
//printf("> %i\n", data_offs);
break;
case SET_ENTR_POINT:
case RUN_PROG:
rel_entr_point = *(uint32_t*)bytes; bytes += 4;
printf("SET_ENTR_POINT rel_entr_point=%i\n", rel_entr_point);
printf("RUN_PROG rel_entr_point=%i\n", rel_entr_point);
entr_point = (int (*)())(executable_memory + rel_entr_point);
mark_mem_executable();
mark_mem_executable(executable_memory, executable_memory_len);
int ret = entr_point();
printf("Return value: %i\n", ret);
break;
case END_PROG:
printf("END_PROG\n");
err_flag = 1;
break;
case READ_DATA:
offs = *(uint32_t*)bytes; bytes += 4;
size = *(uint32_t*)bytes; bytes += 4;
@ -159,6 +124,16 @@ int parse_commands(uint8_t *bytes){
printf("\n");
break;
case FREE_MEMORY:
size = *(uint32_t*)bytes; bytes += 4;
free_memory();
break;
case END_PROG:
printf("END_PROG\n");
err_flag = 1;
break;
default:
printf("Unknown command\n");
err_flag = -1;

2
src/coparun/runmem.h
View File

@ -10,7 +10,7 @@
#define COPY_CODE 4
#define PATCH_FUNC 5
#define PATCH_OBJECT 6
#define SET_ENTR_POINT 64
#define RUN_PROG 64
#define READ_DATA 65
#define END_PROG 256
#define FREE_MEMORY 257

2
src/coparun_module.pyi Normal file
View File

@ -0,0 +1,2 @@
def coparun(data: bytes) -> int: ...
def read_data_mem(rel_addr: int, length: int) -> bytes: ...

2
test.sh
View File

@ -4,6 +4,4 @@ echo "Compile..."
python tests/test_compile.py
echo "Run..."
echo "-----------------------------------"
mkdir bin -p
gcc -Wall -Wextra -Wconversion -Wsign-conversion -Wshadow -Wstrict-overflow -Werror -g src/coparun/runmem.c src/coparun/coparun.c -o bin/coparun
./bin/coparun test.copapy

4
tests/test_compile.py
View File

@ -55,10 +55,10 @@ def test_compile():
r2 = i1 + 9
out = [Write(r1), Write(r2)]
il, _ = copapy.compile_to_instruction_list(out, copapy.sdb)
il, _ = copapy.compile_to_instruction_list(out, copapy.generic_sdb)
# run program command
il.write_com(binwrite.Command.SET_ENTR_POINT)
il.write_com(binwrite.Command.RUN_PROG)
il.write_int(0)
il.write_com(binwrite.Command.READ_DATA)

4
tests/test_coparun_module2.py
View File

@ -14,10 +14,10 @@ def test_compile():
r2 = i1 + 9
out = [Write(r1), Write(r2), Write(c2)]
il, _ = copapy.compile_to_instruction_list(out, copapy.sdb)
il, _ = copapy.compile_to_instruction_list(out, copapy.generic_sdb)
# run program command
il.write_com(binwrite.Command.SET_ENTR_POINT)
il.write_com(binwrite.Command.RUN_PROG)
il.write_int(0)
il.write_com(binwrite.Command.READ_DATA)

7
tests/test_stencil_db.py
View File

@ -1,9 +1,11 @@
from copapy import stencil_database
from copapy import stencil_db
import platform
def test_list_symbols():
sdb = stencil_database('src/copapy/obj/stencils_x86_64_O3.o')
arch = platform.machine()
sdb = stencil_database(f'src/copapy/obj/stencils_{arch}_O3.o')
print('----')
#print(sdb.function_definitions)
for sym_name in sdb.function_definitions.keys():
@ -12,7 +14,8 @@ def test_list_symbols():
def test_start_end_function():
sdb = stencil_database('src/copapy/obj/stencils_x86_64_O3.o')
arch = platform.machine()
sdb = stencil_database(f'src/copapy/obj/stencils_{arch}_O3.o')
for sym_name in sdb.function_definitions.keys():
data = sdb.elf.symbols[sym_name].data
print('-', sym_name, stencil_db.get_stencil_position(data, sdb.elf.byteorder), len(data))