diff --git a/CMakeLists.txt b/CMakeLists.txt index f63146a..8e8cccd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,18 +23,34 @@ option(DTO_NUMA_SUPPORT "Enable NUMA awareness support (requires libnuma)" ON) option(DTO_STATIC_ACCEL_CONFIG "Statically link accel-config into libdto.so" OFF) option(DTO_STATIC_NUMA "Statically link numa into libdto.so (requires PIC-compiled libnuma.a)" OFF) +# libdto_explicit is the same library without the memcpy/memset/memmove/ +# memcmp interposers, for programs that call the dto_* API (dto.h) directly +# and must not have every large libc copy in the process sent to DSA. +option(DTO_BUILD_EXPLICIT "Also build libdto_explicit, without libc interposition" ON) + # Build the shared library add_library(dto SHARED dto.c) add_library(DTO::dto ALIAS dto) -target_include_directories(dto - PUBLIC - $ - $) +set(DTO_LIBRARIES dto) +if(DTO_BUILD_EXPLICIT) + add_library(dto_explicit SHARED dto.c) + add_library(DTO::dto_explicit ALIAS dto_explicit) + target_compile_definitions(dto_explicit PRIVATE DTO_NO_LIBC_INTERPOSE) + list(APPEND DTO_LIBRARIES dto_explicit) +endif() + +foreach(lib ${DTO_LIBRARIES}) + target_include_directories(${lib} + PUBLIC + $ + $) +endforeach() # set gnu source everywhere add_compile_definitions(_GNU_SOURCE) -# Add the -DDTO_STATS_SUPPORT preprocessor definition +# Add the -DDTO_STATS_SUPPORT preprocessor definition. Stats cover the +# interposed calls, so libdto_explicit has none to collect. target_compile_definitions(dto PRIVATE DTO_STATS_SUPPORT) # Link libraries @@ -43,7 +59,9 @@ set(DTO_DYNAMIC_LIBS dl) # Handle accel-config support if(DTO_ACCEL_CONFIG_SUPPORT) - target_compile_definitions(dto PRIVATE DTO_ACCEL_CONFIG_SUPPORT) + foreach(lib ${DTO_LIBRARIES}) + target_compile_definitions(${lib} PRIVATE DTO_ACCEL_CONFIG_SUPPORT) + endforeach() message(STATUS "accel-config support: enabled") if(DTO_STATIC_ACCEL_CONFIG) @@ -62,7 +80,9 @@ endif() # Handle numa support if(DTO_NUMA_SUPPORT) - target_compile_definitions(dto PRIVATE DTO_NUMA_SUPPORT) + foreach(lib ${DTO_LIBRARIES}) + target_compile_definitions(${lib} PRIVATE DTO_NUMA_SUPPORT) + endforeach() message(STATUS "NUMA support: enabled") if(DTO_STATIC_NUMA) @@ -83,21 +103,25 @@ endif() # Link everything together. Use PRIVATE so accel-config/numa/dl are the shared # library's own dependencies (resolved via its DT_NEEDED) rather than INTERFACE # requirements that would be forced onto every downstream consumer of DTO::dto. -if(DTO_STATIC_LIBS) - target_link_libraries(dto PRIVATE - -Wl,--whole-archive - ${DTO_STATIC_LIBS} - -Wl,--no-whole-archive - ${DTO_DYNAMIC_LIBS} - ) -else() - target_link_libraries(dto PRIVATE ${DTO_DYNAMIC_LIBS}) -endif() +foreach(lib ${DTO_LIBRARIES}) + if(DTO_STATIC_LIBS) + target_link_libraries(${lib} PRIVATE + -Wl,--whole-archive + ${DTO_STATIC_LIBS} + -Wl,--no-whole-archive + ${DTO_DYNAMIC_LIBS} + ) + else() + target_link_libraries(${lib} PRIVATE ${DTO_DYNAMIC_LIBS}) + endif() +endforeach() include(CheckCCompilerFlag) check_c_compiler_flag("-mwaitpkg" HAS_WAITPKG) if (HAS_WAITPKG) - target_compile_options(dto PRIVATE -mwaitpkg -march=native) + foreach(lib ${DTO_LIBRARIES}) + target_compile_options(${lib} PRIVATE -mwaitpkg -march=native) + endforeach() endif() # Build dto-test and dto-test-wodto @@ -123,6 +147,18 @@ if(DTO_BUILD_TESTS) TIMEOUT 120 ) + # The same tests against libdto_explicit: the explicit API without the + # interposers (the libc mem* tests then exercise libc itself). + if(DTO_BUILD_EXPLICIT) + add_executable(dto-test-functional-explicit tests/test_functional.c) + target_link_libraries(dto-test-functional-explicit PRIVATE DTO::dto_explicit pthread) + add_test(NAME functional_explicit COMMAND dto-test-functional-explicit) + set_tests_properties(functional_explicit PROPERTIES + LABELS "functional" + TIMEOUT 120 + ) + endif() + # ---- A/B performance regression test ---- # The baseline object is compiled from unguarded upstream sources that # always reference accel-config and numa, so the perf test can only be @@ -225,8 +261,10 @@ if(DTO_BUILD_TESTS) endif() endif() -# Install and export the library -install(TARGETS dto +install(FILES dto.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + +# Install and export the libraries +install(TARGETS ${DTO_LIBRARIES} EXPORT DTOTargets RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/Makefile b/Makefile index 06cc2e1..4da8111 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,13 @@ libdto: dto.c libdto_nostats: dto.c gcc -shared -fPIC -Wl,-soname,libdto.so dto.c $(DML_LIB_CXX) -DDTO_ACCEL_CONFIG_SUPPORT -DDTO_NUMA_SUPPORT -o libdto.so.1.0 -laccel-config -ldl -lnuma -mwaitpkg +# the explicit dto_* API without the libc memcpy/memset/memmove/memcmp interposers +libdto_explicit: dto.c + gcc -shared -fPIC -Wl,-soname,libdto_explicit.so dto.c $(DML_LIB_CXX) -DDTO_NO_LIBC_INTERPOSE -DDTO_ACCEL_CONFIG_SUPPORT -DDTO_NUMA_SUPPORT -o libdto_explicit.so.1.0 -laccel-config -ldl -lnuma -mwaitpkg + install: cp libdto.so.1.0 /usr/lib64/ + cp dto.h /usr/include/ ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so.1 ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so diff --git a/README.md b/README.md index bd3e239..42cda0e 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Following environment variables control the behavior of DTO library: DTO_UMWAIT_DELAY=xxxx defines delay for umwait command (check max possible value at: /sys/devices/system/cpu/umwait_control/max_time), default is 100000 DTO_LOG_FILE= Redirect the DTO output to the specified file instead of std output (useful for debugging and statistics collection). file name is suffixed by process pid. DTO_LOG_LEVEL=0/1/2 controls the log level. higher value means more verbose logging (default 0). + DTO_CRC_MIN_BYTES=xxxx smallest buffer dto_crc32c_with_seed() sends to DSA; smaller ones are checksummed on the CPU (default: DTO_MIN_BYTES). ``` Although not the only usage models of DTO, the following are some common ones: @@ -95,6 +96,51 @@ Although not the only usage models of DTO, the following are some common ones: DTO_WAIT_METHOD=yield or umwait (saves either cycles or power) +## Explicit asynchronous API + +Besides intercepting the libc calls, libdto exports an explicit submit/poll +API, declared in `dto.h`, for applications that want to overlap DSA work with +their own: copy, fill, CRC32C (optionally fused with a copy), compare, compare +against a pattern and translation fetch, each as a single operation or batched +(up to 1024 operations in one DSA batch descriptor). + +```c +#include + +dto_async_op op; /* caller-owned, 64-byte aligned */ + +if (dto_submit_memcpy_crc(&op, dst, src, n, DTO_SUBMIT_CC) == + DTO_ASYNC_SUBMITTED) { + do_other_work(); + dto_async_wait(&op); /* waits with DTO_WAIT_METHOD */ + if (dto_async_poll(&op) == DTO_ASYNC_DONE) + crc = dto_async_crc_val(&op); + else + redo_on_cpu(); +} else { /* DTO_ASYNC_FALLBACK: nothing was done */ + redo_on_cpu(); +} +``` + +A batch is built with `dto_batch_create`, `dto_batch_add_*` and +`dto_batch_submit`; `dto_batch_poll` redoes on the CPU anything the device +could not finish, so once it returns `DTO_ASYNC_DONE` every result is valid. + +Explicit submits are not subject to `DTO_MIN_BYTES`: the caller decides what to +offload. `DTO_SUBMIT_BOF` makes the device wait for page faults to be +resolved; it requires a work queue configured with block-on-fault +(`accel-config config-wq --block-on-fault=1`), and on one without it the +device rejects the operation. Without the flag a page fault stops the +operation partway, on any work queue. `dto_async_crc_val` and `dto_batch_crc` +return the raw CRC32C (the SSE4.2 `crc32` instruction applied from 0, no final +inversion); the seeded `*_crc32c` calls return standard CRC32C and chain across +buffers. See `dto.h` for the full contract. + +Programs that only use this API, and must not have their own large libc copies +sent to DSA, can link `libdto_explicit` instead: the same library without the +memcpy/memset/memmove/memcmp interposers (`-ldto_explicit`, or +`DTO::dto_explicit` from CMake). + ## Build Pre-requisite packages: @@ -124,6 +170,7 @@ The accel-config and libnuma dependencies are optional and can be disabled at co | `DTO_NUMA_SUPPORT` | `ON` | Use libnuma for NUMA-aware work queue selection. When disabled, libdto does not link against libnuma and the `DTO_IS_NUMA_AWARE` environment variable has no effect (work queues are used in round-robin order). | | `DTO_STATIC_ACCEL_CONFIG` | `OFF` | Statically link libaccel-config into libdto.so instead of depending on the shared library at runtime. Requires `libaccel-config.a` to be installed. | | `DTO_STATIC_NUMA` | `OFF` | Statically link libnuma into libdto.so instead of depending on the shared library at runtime. Requires a `libnuma.a` compiled with `-fPIC`. | +| `DTO_BUILD_EXPLICIT` | `ON` | Also build `libdto_explicit`, the explicit API (`dto.h`) without the libc interposers. | Example: build without either dependency: ```bash diff --git a/TESTING.md b/TESTING.md index a491d31..d0e1b37 100644 --- a/TESTING.md +++ b/TESTING.md @@ -59,6 +59,26 @@ hardware -- DTO falls back to CPU transparently. ctest -R functional ``` +They also cover the explicit asynchronous API (`dto.h`): every submit and +batch operation is checked against the CPU result, including the CPU repair of +batch operations the device could not finish. `functional_explicit` runs the +same tests against `libdto_explicit`. + +Because every explicit operation may legitimately fall back to the CPU, a +broken offload path would still pass. On a machine with a usable work queue, +set `DTO_TEST_EXPECT_DSA` to require that operations on faulted-in buffers are +submitted and completed by the device: + +```bash +sudo DTO_TEST_EXPECT_DSA=1 ./dto-test-functional +sudo DTO_TEST_EXPECT_DSA=1 ./dto-test-functional-explicit +``` + +`test_async_bof` checks the block-on-fault behavior against whichever setting +the work queue has (it reports which), so run the suite once with +`block_on_fault` enabled and once without to cover both. It assumes all work +queues DTO uses share the same setting. + ## Performance Tests The performance test (`perf`) detects regressions in libdto by diff --git a/dto.c b/dto.c index 5eb211e..24b606b 100644 --- a/dto.c +++ b/dto.c @@ -16,6 +16,8 @@ #include #include #include + +#include "dto.h" #include #include #include @@ -64,7 +66,9 @@ #define NSEC_PER_MSEC (NSEC_PER_SEC/MSEC_PER_SEC) // thread specific variables +#ifndef DTO_NO_LIBC_INTERPOSE static __thread struct dsa_hw_desc thr_desc; +#endif static __thread struct dsa_completion_record thr_comp __attribute__((aligned(32))); static __thread uint64_t thr_bytes_completed; @@ -82,6 +86,7 @@ struct dto_wq { uint64_t dsa_gencap; int wq_size; uint32_t max_transfer_size; + uint32_t max_batch_size; int wq_fd; void *wq_portal; bool wq_mmapped; @@ -133,6 +138,9 @@ static atomic_uchar dto_initializing; static uint8_t use_std_lib_calls; static enum numa_aware is_numa_aware; static size_t dsa_min_size = DTO_DEFAULT_MIN_SIZE; +/* DTO_CRC_MIN_BYTES: smallest dto_crc32c_with_seed() sent to DSA; 0 follows + * dsa_min_size */ +static size_t crc_min_size; static int wait_method = WAIT_BUSYPOLL; static size_t cpu_size_fraction; // range of values is 0 to 99 @@ -998,6 +1006,12 @@ static int dsa_init_from_wq_list(char *wq_list) goto fail_wq; } + wqs[num_wqs].max_batch_size = dto_get_param_ullong(dir_fd, "max_batch_size", &rc); + if (rc) { + close(dir_fd); + goto fail_wq; + } + dto_get_param_string(dir_fd, "mode", wq_mode); if (wq_mode[0] == '\0') { @@ -1161,6 +1175,7 @@ static int dsa_init_from_accfg(void) wqs[num_wqs].wq_size = accfg_wq_get_size(wq); wqs[num_wqs].max_transfer_size = accfg_wq_get_max_transfer_size(wq); + wqs[num_wqs].max_batch_size = accfg_wq_get_max_batch_size(wq); wqs[num_wqs].acc_wq = wq; wqs[num_wqs].dsa_gencap = accfg_device_get_gen_cap(device); @@ -1484,6 +1499,15 @@ static int init_dto(void) dsa_min_size = DTO_DEFAULT_MIN_SIZE; } + env_str = getenv("DTO_CRC_MIN_BYTES"); + + if (env_str != NULL) { + errno = 0; + crc_min_size = strtoul(env_str, NULL, 10); + if (errno) + crc_min_size = 0; + } + double cpu_size_fraction_float = 0.0; env_str = getenv("DTO_CPU_SIZE_FRACTION"); @@ -1630,15 +1654,23 @@ static __always_inline struct dto_wq *get_wq(void* buf) return wq; } +/* 8-byte MEMFILL pattern for the byte value c (like memset) */ +static __always_inline uint64_t memfill_pattern(int c) +{ + return 0x0101010101010101ULL * (uint8_t)c; +} + +/* libdto_explicit (DTO_NO_LIBC_INTERPOSE) leaves libc's memset, memcpy, + * memmove and memcmp alone: programs that use it call the dto_* API below + * directly. */ +#ifndef DTO_NO_LIBC_INTERPOSE + static void dto_memset(void *s, int c, size_t n, int *result) { - uint64_t memset_pattern; + uint64_t memset_pattern = memfill_pattern(c); size_t cpu_size, dsa_size; struct dto_wq *wq = get_wq(s); - for (int i = 0; i < 8; ++i) - ((uint8_t *) &memset_pattern)[i] = (uint8_t) c; - thr_desc.opcode = DSA_OPCODE_MEMFILL; thr_desc.flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; if (dto_dsa_cc && (wq->dsa_gencap & GENCAP_CC_MEMORY)) @@ -2150,3 +2182,866 @@ int memcmp(const void *s1, const void *s2, size_t n) } return ret; } + +#endif /* DTO_NO_LIBC_INTERPOSE */ + +/******************************************************************************* + * Explicit asynchronous submit/poll API (see dto.h). + ******************************************************************************/ + +/* CRC presentations. With its default CRC flags the device inverts both the + * seed and the result: the iSCSI presentation, in which seed 0 gives the + * standard CRC32C and a previous result chains. The crc32c calls return it + * unchanged. The raw CRC32C of dto_async_crc_val/dto_batch_crc -- the SSE4.2 + * crc32 loop from 0, no final inversion -- is the same computation seeded + * with ~0 and inverted back. */ +#define DSA_CRC_SEED_RAW 0xFFFFFFFFu +#define DSA_CRC_VAL_TO_RAW(v) ((uint32_t)~(uint32_t)(v)) + +/* ENQCMD to a shared WQ can transiently fail when the queue is full; + * retry briefly before giving up so momentary bursts don't push work + * back onto the CPU. A batch carries more work, so it tries longer. */ +#define ASYNC_SUBMIT_RETRIES 16 +#define BATCH_SUBMIT_RETRIES 64 + +/* Failed async completions are logged at most this many times per process: + * poll may legitimately be called repeatedly on a failed op. */ +#define ASYNC_FAIL_LOG_LIMIT 3 + +/* dto_async_wait() must not wait for a status byte no device will ever + * write: an op whose submit fell back, or one never submitted at all (ops + * are usually stack variables, so their storage starts out as garbage). + * Every submit clears the mark first and sets it once the descriptor is + * enqueued; garbage matching it by chance is a 2^-32 event. */ +#define DTO_OP_SUBMITTED_MAGIC 0x64746F53u /* "dtoS" */ + +struct dto_async_op_impl { + struct dsa_hw_desc desc; /* 64 bytes, 64-aligned via dto_async_op */ + struct dsa_completion_record comp __attribute__((aligned(32))); + uint32_t submitted; +}; +_Static_assert(sizeof(struct dto_async_op_impl) <= sizeof(dto_async_op), + "dto_async_op opaque storage too small"); +_Static_assert(sizeof(struct dsa_hw_desc) == 64, "unexpected descriptor size"); + +typedef uint64_t __attribute__((may_alias, aligned(1))) unaligned_u64; + +/* CRC32C on the CPU in the device's presentation for the given seed, so a + * value the library computes matches what the device returns for the same + * descriptor. */ +__attribute__((target("sse4.2"))) +static uint32_t cpu_crc32c(uint32_t seed, const void *buf, size_t n) +{ + const uint8_t *p = buf; + uint64_t crc = (uint32_t)~seed; + + for (; n >= 8; n -= 8, p += 8) + crc = _mm_crc32_u64(crc, *(const unaligned_u64 *)p); + for (; n; n--) + crc = _mm_crc32_u8((uint32_t)crc, *p++); + return ~(uint32_t)crc; +} + +static __always_inline uint32_t async_desc_flags(const struct dto_wq *wq, + unsigned int flags, + bool has_dest) +{ + uint32_t f = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; + + /* Passed as given: a work queue configured without block-on-fault + * fails the descriptor with DSA_COMP_INVALID_FLAGS, and without the + * flag a page fault stops the operation partway (see dto.h). */ + if (flags & DTO_SUBMIT_BOF) + f |= IDXD_OP_FLAG_BOF; + /* CC is only legal for operations with a destination; CRC Generation + * would be failed by the device with DSA_COMP_INVALID_FLAGS. */ + if ((flags & DTO_SUBMIT_CC) && has_dest && + (wq->dsa_gencap & GENCAP_CC_MEMORY)) + f |= IDXD_OP_FLAG_CC; + return f; +} + +static int async_submit(struct dto_wq *wq, struct dsa_hw_desc *desc, + int retries) +{ + for (int attempt = 0; attempt < retries; attempt++) { + int rc = dsa_submit(wq, desc); + + if (rc == SUCCESS) + return DTO_ASYNC_SUBMITTED; + if (rc != RETRY) + break; + _mm_pause(); + } + return DTO_ASYNC_FALLBACK; +} + +static void async_log_failure(uint8_t status, uint32_t opcode, uint32_t size) +{ + static int fail_logged; + + if (fail_logged < ASYNC_FAIL_LOG_LIMIT && + __atomic_fetch_add(&fail_logged, 1, __ATOMIC_RELAXED) < + ASYNC_FAIL_LOG_LIMIT) + LOG_ERROR("async op failed with status %x (opcode %x, size %x)\n", + status, opcode, size); +} + +/* Completion record status -> DTO_ASYNC_{PENDING,DONE,FAILED} */ +static int async_comp_result(const struct dsa_completion_record *comp, + uint32_t opcode, uint32_t size) +{ + uint8_t status = __atomic_load_n((const uint8_t *)&comp->status, + __ATOMIC_ACQUIRE); + + if (status == 0) + return DTO_ASYNC_PENDING; + if (likely((status & DSA_COMP_STATUS_MASK) == DSA_COMP_SUCCESS)) + return DTO_ASYNC_DONE; + async_log_failure(status, opcode, size); + return DTO_ASYNC_FAILED; +} + +/* Common part of every single-op submit. NULL means fall back, with nothing + * submitted; otherwise the descriptor is set up but for its operands. hint + * picks the work queue (NUMA-aware selection uses its node). */ +static struct dto_wq *async_prepare(struct dto_async_op_impl *impl, + uint32_t opcode, const void *hint, + size_t n, unsigned int flags, bool has_dest) +{ + struct dto_wq *wq; + + impl->submitted = 0; + if (!dto_initialized || use_std_lib_calls) + return NULL; + /* The completion record must be 32-byte aligned. dto_async_op is + * declared 64-byte aligned, but a malloc'ed op need not be. */ + if (n == 0 || ((uintptr_t)impl & 31) != 0) + return NULL; + wq = get_wq((void *)hint); + if (unlikely(wq == NULL) || n > wq->max_transfer_size) + return NULL; + + orig_memset(&impl->desc, 0, sizeof(impl->desc)); + impl->desc.opcode = opcode; + impl->desc.flags = async_desc_flags(wq, flags, has_dest); + impl->desc.completion_addr = (uint64_t)&impl->comp; + impl->desc.xfer_size = (uint32_t)n; + impl->comp.status = 0; + return wq; +} + +static int async_enqueue(struct dto_async_op_impl *impl, struct dto_wq *wq) +{ + if (async_submit(wq, &impl->desc, ASYNC_SUBMIT_RETRIES) != + DTO_ASYNC_SUBMITTED) + return DTO_ASYNC_FALLBACK; + impl->submitted = DTO_OP_SUBMITTED_MAGIC; + return DTO_ASYNC_SUBMITTED; +} + +__attribute__((visibility("default"))) +int dto_submit_memcpy(dto_async_op *op, void *dest, const void *src, + size_t n, unsigned int flags) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + struct dto_wq *wq = async_prepare(impl, DSA_OPCODE_MEMMOVE, dest, n, + flags, true); + + if (wq == NULL) + return DTO_ASYNC_FALLBACK; + impl->desc.src_addr = (uint64_t)src; + impl->desc.dst_addr = (uint64_t)dest; + return async_enqueue(impl, wq); +} + +__attribute__((visibility("default"))) +int dto_submit_memfill(dto_async_op *op, void *dest, uint64_t pattern, + size_t n, unsigned int flags) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + struct dto_wq *wq = async_prepare(impl, DSA_OPCODE_MEMFILL, dest, n, + flags, true); + + if (wq == NULL) + return DTO_ASYNC_FALLBACK; + impl->desc.pattern = pattern; + impl->desc.dst_addr = (uint64_t)dest; + return async_enqueue(impl, wq); +} + +__attribute__((visibility("default"))) +int dto_submit_memset(dto_async_op *op, void *dest, int c, size_t n, + unsigned int flags) +{ + return dto_submit_memfill(op, dest, memfill_pattern(c), n, flags); +} + +static int submit_crc_common(dto_async_op *op, uint32_t opcode, void *dest, + const void *src, size_t n, uint32_t seed, + unsigned int flags) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + struct dto_wq *wq = async_prepare(impl, opcode, dest ? dest : src, n, + flags, dest != NULL); + + if (wq == NULL) + return DTO_ASYNC_FALLBACK; + impl->desc.src_addr = (uint64_t)src; + impl->desc.dst_addr = (uint64_t)dest; + impl->desc.crc_seed = seed; + return async_enqueue(impl, wq); +} + +__attribute__((visibility("default"))) +int dto_submit_memcpy_crc(dto_async_op *op, void *dest, const void *src, + size_t n, unsigned int flags) +{ + return submit_crc_common(op, DSA_OPCODE_COPY_CRC, dest, src, n, + DSA_CRC_SEED_RAW, flags); +} + +__attribute__((visibility("default"))) +int dto_submit_crc(dto_async_op *op, const void *src, size_t n, + unsigned int flags) +{ + return submit_crc_common(op, DSA_OPCODE_CRCGEN, NULL, src, n, + DSA_CRC_SEED_RAW, flags); +} + +__attribute__((visibility("default"))) +int dto_submit_memcpy_crc32c(dto_async_op *op, void *dest, const void *src, + size_t n, uint32_t seed, unsigned int flags) +{ + return submit_crc_common(op, DSA_OPCODE_COPY_CRC, dest, src, n, seed, + flags); +} + +__attribute__((visibility("default"))) +int dto_submit_crc32c(dto_async_op *op, const void *src, size_t n, + uint32_t seed, unsigned int flags) +{ + return submit_crc_common(op, DSA_OPCODE_CRCGEN, NULL, src, n, seed, + flags); +} + +__attribute__((visibility("default"))) +int dto_submit_compare(dto_async_op *op, const void *src1, const void *src2, + size_t n, unsigned int flags) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + struct dto_wq *wq = async_prepare(impl, DSA_OPCODE_COMPARE, src1, n, + flags, false); + + if (wq == NULL) + return DTO_ASYNC_FALLBACK; + impl->desc.src_addr = (uint64_t)src1; + impl->desc.src2_addr = (uint64_t)src2; + return async_enqueue(impl, wq); +} + +__attribute__((visibility("default"))) +int dto_submit_compval(dto_async_op *op, const void *src, uint64_t pattern, + size_t n, unsigned int flags) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + struct dto_wq *wq = async_prepare(impl, DSA_OPCODE_COMPVAL, src, n, + flags, false); + + if (wq == NULL) + return DTO_ASYNC_FALLBACK; + impl->desc.src_addr = (uint64_t)src; + /* Compare Pattern takes its pattern from the second operand slot; + * the first one is the source (it is Memory Fill's pattern slot). */ + impl->desc.comp_pattern = pattern; + return async_enqueue(impl, wq); +} + +__attribute__((visibility("default"))) +int dto_submit_transl_fetch(dto_async_op *op, const void *addr, size_t n, + unsigned int flags) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + struct dto_wq *wq = async_prepare(impl, DSA_OPCODE_TRANSL_FETCH, addr, + n, flags, false); + + if (wq == NULL) + return DTO_ASYNC_FALLBACK; + /* region_size shares the xfer_size slot async_prepare filled in */ + impl->desc.transl_fetch_addr = (uint64_t)addr; + impl->desc.region_stride = 0; /* one contiguous region */ + return async_enqueue(impl, wq); +} + +__attribute__((visibility("default"))) +int dto_async_poll(dto_async_op *op) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + + return async_comp_result(&impl->comp, impl->desc.opcode, + impl->desc.xfer_size); +} + +__attribute__((visibility("default"))) +void dto_async_wait(dto_async_op *op) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + + if (impl->submitted != DTO_OP_SUBMITTED_MAGIC) + return; + dsa_wait_no_adjust(&impl->comp.status); + /* order the status byte ahead of the caller's reads of the record */ + atomic_thread_fence(memory_order_acquire); +} + +__attribute__((visibility("default"))) +int dto_wait_blocks(void) +{ + return 0; +} + +__attribute__((visibility("default"))) +uint32_t dto_async_crc_val(const dto_async_op *op) +{ + const struct dto_async_op_impl *impl = + (const struct dto_async_op_impl *)op; + return DSA_CRC_VAL_TO_RAW(impl->comp.crc_val); +} + +__attribute__((visibility("default"))) +uint32_t dto_async_crc32c_val(const dto_async_op *op) +{ + const struct dto_async_op_impl *impl = + (const struct dto_async_op_impl *)op; + return (uint32_t)impl->comp.crc_val; +} + +__attribute__((visibility("default"))) +int dto_async_result(const dto_async_op *op) +{ + const struct dto_async_op_impl *impl = + (const struct dto_async_op_impl *)op; + return impl->comp.result; +} + +__attribute__((visibility("default"))) +uint32_t dto_async_bytes_completed(const dto_async_op *op) +{ + const struct dto_async_op_impl *impl = + (const struct dto_async_op_impl *)op; + return impl->comp.bytes_completed; +} + +__attribute__((visibility("default"))) +int dto_async_status(const dto_async_op *op) +{ + const struct dto_async_op_impl *impl = + (const struct dto_async_op_impl *)op; + return __atomic_load_n((const uint8_t *)&impl->comp.status, + __ATOMIC_ACQUIRE) & DSA_COMP_STATUS_MASK; +} + +__attribute__((visibility("default"))) +uint32_t dto_crc32c_with_seed(uint32_t seed, const void *src, size_t n) +{ + size_t min_size = crc_min_size ? crc_min_size : dsa_min_size; + dto_async_op op; + + /* No DTO_SUBMIT_BOF, so this works on any work queue: a page fault + * stops the device and the CRC is computed on the CPU instead. */ + if (n >= min_size && + dto_submit_crc32c(&op, src, n, seed, 0) == DTO_ASYNC_SUBMITTED) { + dto_async_wait(&op); + if (dto_async_poll(&op) == DTO_ASYNC_DONE) + return dto_async_crc32c_val(&op); + } + return cpu_crc32c(seed, src, n); +} + +/* ---- Batches (see dto.h) ---- */ + +/* Zero storage the library owns. Once DTO is up that is libc's memset, so it + * costs no detour through the interposer and a tiny DTO_MIN_BYTES never sends + * it to the device; before that, memset() itself, which copes with an + * uninitialized DTO (and is libc's in libdto_explicit). */ +static void lib_zero(void *p, size_t n) +{ + if (orig_memset) + orig_memset(p, 0, n); + else + memset(p, 0, n); +} + +enum batch_state { + BATCH_BUILDING = 0, /* accepting operations */ + BATCH_SUBMITTED, /* in flight */ + BATCH_REPAIRING, /* one poller is redoing failed operations */ + BATCH_DONE, +}; + +struct dto_batch { + struct dsa_hw_desc desc __attribute__((aligned(64))); /* the Batch descriptor */ + struct dsa_completion_record comp __attribute__((aligned(32))); + struct dsa_hw_desc *descs; /* [cap], 64-byte aligned */ + struct dsa_completion_record *comps; /* [cap], 32-byte aligned */ + const void *hint; /* first operand: picks the WQ */ + int cap; + int count; + _Atomic int state; +}; + +/* The record that completes the batch: the Batch descriptor's, or for a + * single operation, sent as a plain descriptor, that operation's own. */ +static __always_inline volatile uint8_t *batch_status(dto_batch *b) +{ + return b->count == 1 ? &b->comps[0].status : &b->comp.status; +} + +/* Wait out a batch the device may still be writing to. */ +static void batch_quiesce(dto_batch *b) +{ + if (atomic_load_explicit(&b->state, memory_order_acquire) == + BATCH_SUBMITTED) + dsa_wait_no_adjust(batch_status(b)); + while (atomic_load_explicit(&b->state, memory_order_acquire) == + BATCH_REPAIRING) + _mm_pause(); +} + +__attribute__((visibility("default"))) +dto_batch *dto_batch_create(int capacity) +{ + void *b = NULL, *descs = NULL, *comps = NULL; + + if (capacity < 1) + return NULL; + if (capacity > DTO_BATCH_MAX) + capacity = DTO_BATCH_MAX; + if (posix_memalign(&b, 64, sizeof(struct dto_batch)) || + posix_memalign(&descs, 64, sizeof(struct dsa_hw_desc) * capacity) || + posix_memalign(&comps, 32, + sizeof(struct dsa_completion_record) * capacity)) { + free(b); + free(descs); + return NULL; + } + /* operations zero their own descriptor and record when added */ + lib_zero(b, sizeof(struct dto_batch)); + ((struct dto_batch *)b)->descs = descs; + ((struct dto_batch *)b)->comps = comps; + ((struct dto_batch *)b)->cap = capacity; + return b; +} + +__attribute__((visibility("default"))) +void dto_batch_destroy(dto_batch *b) +{ + if (b == NULL) + return; + batch_quiesce(b); + free(b->descs); + free(b->comps); + free(b); +} + +__attribute__((visibility("default"))) +void dto_batch_reset(dto_batch *b) +{ + batch_quiesce(b); + b->count = 0; + b->hint = NULL; + atomic_store_explicit(&b->state, BATCH_BUILDING, memory_order_relaxed); +} + +__attribute__((visibility("default"))) +int dto_batch_count(const dto_batch *b) +{ + return b->count; +} + +/* Descriptor for the next operation, or NULL (with *index unset) when the + * batch is full or submitted or n is out of range. BOF is set as requested; + * CC is recorded as requested and settled at submit, once the work queue + * (and so the device's cache-control support) is known. */ +static struct dsa_hw_desc *batch_add(dto_batch *b, uint32_t opcode, + const void *hint, size_t n, + unsigned int flags, bool has_dest, + int *index) +{ + struct dsa_hw_desc *d; + int i; + + if (atomic_load_explicit(&b->state, memory_order_relaxed) != + BATCH_BUILDING || b->count >= b->cap || n == 0 || n > UINT32_MAX) + return NULL; + i = b->count++; + d = &b->descs[i]; + lib_zero(d, sizeof(*d)); + d->opcode = opcode; + d->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; + if (flags & DTO_SUBMIT_BOF) + d->flags |= IDXD_OP_FLAG_BOF; + if ((flags & DTO_SUBMIT_CC) && has_dest) + d->flags |= IDXD_OP_FLAG_CC; + d->completion_addr = (uint64_t)&b->comps[i]; + d->xfer_size = (uint32_t)n; + lib_zero(&b->comps[i], sizeof(b->comps[i])); + if (b->hint == NULL) + b->hint = hint; + *index = i; + return d; +} + +__attribute__((visibility("default"))) +int dto_batch_add_memmove(dto_batch *b, void *dst, const void *src, size_t n, + unsigned int flags) +{ + int i; + struct dsa_hw_desc *d = batch_add(b, DSA_OPCODE_MEMMOVE, dst, n, flags, + true, &i); + + if (d == NULL) + return -1; + d->src_addr = (uint64_t)src; + d->dst_addr = (uint64_t)dst; + return i; +} + +__attribute__((visibility("default"))) +int dto_batch_add_memfill(dto_batch *b, void *dst, uint64_t pattern, size_t n, + unsigned int flags) +{ + int i; + struct dsa_hw_desc *d = batch_add(b, DSA_OPCODE_MEMFILL, dst, n, flags, + true, &i); + + if (d == NULL) + return -1; + d->pattern = pattern; + d->dst_addr = (uint64_t)dst; + return i; +} + +__attribute__((visibility("default"))) +int dto_batch_add_dualcast(dto_batch *b, void *dst1, void *dst2, + const void *src, size_t n, unsigned int flags) +{ + struct dsa_hw_desc *d; + int i; + + /* the device requires both destinations to agree in bits 11:0 */ + if ((((uintptr_t)dst1 ^ (uintptr_t)dst2) & 0xfff) != 0) + return -1; + d = batch_add(b, DSA_OPCODE_DUALCAST, dst1, n, flags, true, &i); + if (d == NULL) + return -1; + d->src_addr = (uint64_t)src; + d->dst_addr = (uint64_t)dst1; + d->dest2 = (uint64_t)dst2; + return i; +} + +__attribute__((visibility("default"))) +int dto_batch_add_crc(dto_batch *b, const void *src, size_t n, + unsigned int flags) +{ + int i; + struct dsa_hw_desc *d = batch_add(b, DSA_OPCODE_CRCGEN, src, n, flags, + false, &i); + + if (d == NULL) + return -1; + d->src_addr = (uint64_t)src; + d->crc_seed = DSA_CRC_SEED_RAW; + return i; +} + +__attribute__((visibility("default"))) +int dto_batch_add_memcpy_crc(dto_batch *b, void *dst, const void *src, + size_t n, unsigned int flags) +{ + int i; + struct dsa_hw_desc *d = batch_add(b, DSA_OPCODE_COPY_CRC, dst, n, flags, + true, &i); + + if (d == NULL) + return -1; + d->src_addr = (uint64_t)src; + d->dst_addr = (uint64_t)dst; + d->crc_seed = DSA_CRC_SEED_RAW; + return i; +} + +__attribute__((visibility("default"))) +int dto_batch_add_compare(dto_batch *b, const void *src1, const void *src2, + size_t n, unsigned int flags) +{ + int i; + struct dsa_hw_desc *d = batch_add(b, DSA_OPCODE_COMPARE, src1, n, flags, + false, &i); + + if (d == NULL) + return -1; + d->src_addr = (uint64_t)src1; + d->src2_addr = (uint64_t)src2; + return i; +} + +__attribute__((visibility("default"))) +int dto_batch_add_compval(dto_batch *b, const void *src, uint64_t pattern, + size_t n, unsigned int flags) +{ + int i; + struct dsa_hw_desc *d = batch_add(b, DSA_OPCODE_COMPVAL, src, n, flags, + false, &i); + + if (d == NULL) + return -1; + d->src_addr = (uint64_t)src; + d->comp_pattern = pattern; + return i; +} + +__attribute__((visibility("default"))) +int dto_batch_submit(dto_batch *b) +{ + struct dsa_hw_desc *desc; + struct dto_wq *wq; + + if (atomic_load_explicit(&b->state, memory_order_relaxed) != + BATCH_BUILDING) + return DTO_ASYNC_FALLBACK; + if (b->count == 0) { + atomic_store_explicit(&b->state, BATCH_DONE, + memory_order_release); + return DTO_ASYNC_SUBMITTED; + } + if (!dto_initialized || use_std_lib_calls) + return DTO_ASYNC_FALLBACK; + wq = get_wq((void *)b->hint); + if (unlikely(wq == NULL) || + (b->count > 1 && (uint32_t)b->count > wq->max_batch_size)) + return DTO_ASYNC_FALLBACK; + for (int i = 0; i < b->count; i++) { + if (b->descs[i].xfer_size > wq->max_transfer_size) + return DTO_ASYNC_FALLBACK; + } + + for (int i = 0; i < b->count; i++) { + struct dsa_hw_desc *d = &b->descs[i]; + + if (!(wq->dsa_gencap & GENCAP_CC_MEMORY)) + d->flags &= ~IDXD_OP_FLAG_CC; + b->comps[i].status = 0; + } + if (b->count == 1) { + desc = &b->descs[0]; + } else { + orig_memset(&b->desc, 0, sizeof(b->desc)); + b->desc.opcode = DSA_OPCODE_BATCH; + b->desc.flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; + b->desc.desc_list_addr = (uint64_t)b->descs; + b->desc.desc_count = (uint32_t)b->count; + b->desc.completion_addr = (uint64_t)&b->comp; + b->comp.status = 0; + desc = &b->desc; + } + + if (async_submit(wq, desc, BATCH_SUBMIT_RETRIES) != DTO_ASYNC_SUBMITTED) + return DTO_ASYNC_FALLBACK; + atomic_store_explicit(&b->state, BATCH_SUBMITTED, memory_order_release); + return DTO_ASYNC_SUBMITTED; +} + +/* Fill n bytes with the 8-byte pattern, as Memory Fill does. */ +static void cpu_fill(void *dst, uint64_t pattern, size_t n) +{ + uint8_t *p = dst; + size_t i = 0; + + if (pattern == memfill_pattern((uint8_t)pattern)) { + orig_memset(dst, (uint8_t)pattern, n); + return; + } + for (; i + 8 <= n; i += 8) + *(unaligned_u64 *)(p + i) = pattern; + for (; i < n; i++) + p[i] = (uint8_t)(pattern >> (8 * (i & 7))); +} + +/* Fill in a compare's result the way the device does: result 1 and the + * offset of the first difference, or result 0 with every byte compared. + * With src2 NULL, src1 is compared against the 8-byte pattern instead. */ +static void cpu_compare(struct dsa_completion_record *comp, const void *src1, + const void *src2, uint64_t pattern, size_t n) +{ + const uint8_t *a = src1, *b = src2; + size_t i = 0; + + /* a word at a time up to the first differing word, then bytewise */ + for (; i + 8 <= n; i += 8) { + uint64_t want = b ? *(const unaligned_u64 *)(b + i) : pattern; + + if (*(const unaligned_u64 *)(a + i) != want) + break; + } + for (; i < n; i++) { + uint8_t want = b ? b[i] : (uint8_t)(pattern >> (8 * (i & 7))); + + if (a[i] != want) + break; + } + comp->result = i < n; + comp->bytes_completed = (uint32_t)i; +} + +/* A move the device stopped partway (a page fault without BOF) cannot be + * redone from the start when the buffers overlap: the device has already + * overwritten part of the source. bytes_completed counts from the end the + * copy started at, which bit 0 of result gives (1: descending). */ +static void cpu_redo_memmove(struct dsa_completion_record *comp, void *dst, + const void *src, size_t n) +{ + size_t done = comp->bytes_completed; + + if ((comp->status & DSA_COMP_STATUS_MASK) != DSA_COMP_PAGE_FAULT_NOBOF || + done >= n) + done = 0; + if (comp->result & 1) + orig_memmove(dst, src, n - done); + else + orig_memmove((uint8_t *)dst + done, (const uint8_t *)src + done, + n - done); +} + +/* Redo every operation the device did not complete on the CPU. Each status + * is left as the device wrote it, so dto_batch_status() still says who did + * the work; result, bytes_completed and crc_val are filled in as the device + * would have. */ +static void batch_repair(dto_batch *b) +{ + for (int i = 0; i < b->count; i++) { + const struct dsa_hw_desc *d = &b->descs[i]; + struct dsa_completion_record *c = &b->comps[i]; + void *dst = (void *)(uintptr_t)d->dst_addr; + const void *src = (const void *)(uintptr_t)d->src_addr; + size_t n = d->xfer_size; + + if ((c->status & DSA_COMP_STATUS_MASK) == DSA_COMP_SUCCESS) + continue; + switch (d->opcode) { + case DSA_OPCODE_MEMMOVE: + cpu_redo_memmove(c, dst, src, n); + break; + case DSA_OPCODE_MEMFILL: + cpu_fill(dst, d->pattern, n); + break; + case DSA_OPCODE_DUALCAST: + orig_memcpy(dst, src, n); + orig_memcpy((void *)(uintptr_t)d->dest2, src, n); + break; + case DSA_OPCODE_COPY_CRC: + orig_memcpy(dst, src, n); + c->crc_val = cpu_crc32c(d->crc_seed, src, n); + break; + case DSA_OPCODE_CRCGEN: + c->crc_val = cpu_crc32c(d->crc_seed, src, n); + break; + case DSA_OPCODE_COMPARE: + cpu_compare(c, src, (const void *)(uintptr_t)d->src2_addr, + 0, n); + break; + case DSA_OPCODE_COMPVAL: + cpu_compare(c, src, NULL, d->comp_pattern, n); + break; + } + } +} + +__attribute__((visibility("default"))) +int dto_batch_poll(dto_batch *b) +{ + int state = atomic_load_explicit(&b->state, memory_order_acquire); + uint8_t status; + + if (state == BATCH_DONE) + return DTO_ASYNC_DONE; + if (state == BATCH_BUILDING) + return DTO_ASYNC_FAILED; + if (state == BATCH_REPAIRING) + return DTO_ASYNC_PENDING; + + status = __atomic_load_n(batch_status(b), __ATOMIC_ACQUIRE); + if (status == 0) + return DTO_ASYNC_PENDING; + if ((status & DSA_COMP_STATUS_MASK) != DSA_COMP_SUCCESS) { + /* exactly one poller repairs; the others see PENDING meanwhile */ + if (!atomic_compare_exchange_strong(&b->state, &state, + BATCH_REPAIRING)) + return state == BATCH_DONE ? DTO_ASYNC_DONE : + DTO_ASYNC_PENDING; + async_log_failure(status, b->count == 1 ? b->descs[0].opcode : + DSA_OPCODE_BATCH, b->count); + batch_repair(b); + } + atomic_store_explicit(&b->state, BATCH_DONE, memory_order_release); + return DTO_ASYNC_DONE; +} + +__attribute__((visibility("default"))) +void dto_batch_wait(dto_batch *b) +{ + if (atomic_load_explicit(&b->state, memory_order_acquire) != + BATCH_SUBMITTED) + return; + dsa_wait_no_adjust(batch_status(b)); + atomic_thread_fence(memory_order_acquire); +} + +__attribute__((visibility("default"))) +int dto_batch_status(const dto_batch *b, int i) +{ + if (i < 0 || i >= b->count) + return 0; + return __atomic_load_n((const uint8_t *)&b->comps[i].status, + __ATOMIC_ACQUIRE) & DSA_COMP_STATUS_MASK; +} + +__attribute__((visibility("default"))) +int dto_batch_result(const dto_batch *b, int i) +{ + if (i < 0 || i >= b->count) + return 0; + return b->comps[i].result; +} + +__attribute__((visibility("default"))) +uint32_t dto_batch_bytes_completed(const dto_batch *b, int i) +{ + if (i < 0 || i >= b->count) + return 0; + return b->comps[i].bytes_completed; +} + +__attribute__((visibility("default"))) +uint32_t dto_batch_crc(const dto_batch *b, int i) +{ + if (i < 0 || i >= b->count) + return 0; + return DSA_CRC_VAL_TO_RAW(b->comps[i].crc_val); +} + +__attribute__((visibility("default"))) +int dto_submit_batch_copy(dto_batch *b, void **dst, void **src, + size_t *sizes, int count, unsigned int flags) +{ + if (count < 1 || count > b->cap) + return DTO_ASYNC_FALLBACK; + dto_batch_reset(b); + for (int i = 0; i < count; i++) { + if (dto_batch_add_memmove(b, dst[i], src[i], sizes[i], + flags) < 0) + return DTO_ASYNC_FALLBACK; + } + return dto_batch_submit(b); +} diff --git a/dto.h b/dto.h new file mode 100644 index 0000000..9fe58fe --- /dev/null +++ b/dto.h @@ -0,0 +1,275 @@ +/******************************************************************************* + * Copyright (C) 2026 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + * dto.h - Public asynchronous offload API for libdto. + * + * In addition to transparently accelerating memcpy/memmove/memset/memcmp via + * LD_PRELOAD, libdto exposes an explicit submit/poll API for applications + * that want to overlap accelerator work with CPU work. libdto_explicit is + * the same library without the libc interposers, for programs that only call + * this API. + * + * Lifecycle: + * + * dto_async_op op; + * if (dto_submit_memcpy(&op, dst, src, n, DTO_SUBMIT_CC) == DTO_ASYNC_SUBMITTED) { + * ... other CPU work ... + * dto_async_wait(&op); // optional; or poll in your own loop + * while (dto_async_poll(&op) == DTO_ASYNC_PENDING) + * ; // pause/yield + * if (dto_async_poll(&op) != DTO_ASYNC_DONE) + * memcpy(dst, src, n); // redo on the CPU + * } else { // DTO_ASYNC_FALLBACK: nothing submitted + * memcpy(dst, src, n); + * } + * + * The operation state is caller-allocated, so it can outlive the submitting + * call and be polled from a different thread; nothing is stored in + * thread-local state. dto_async_op is declared 64-byte aligned, which + * automatic and static storage honour; heap-allocated ops must come from + * aligned_alloc/posix_memalign (plain malloc does not guarantee it). An op + * that is not at least 32-byte aligned is rejected with DTO_ASYNC_FALLBACK. + * + * Submission returns DTO_ASYNC_FALLBACK when the accelerator is unavailable + * (not initialized, disabled via DTO_USESTDC_CALLS, no usable work queue) or + * the request is out of range (n == 0, or larger than the work queue's + * maximum transfer size). Nothing is done in that case: the caller performs + * the operation on the CPU. Unlike the transparent path, explicit submits are + * not subject to the size heuristics (DTO_MIN_BYTES / auto-tuning): callers + * decide what to offload. + * + * On DTO_ASYNC_FAILED the device did not complete the operation; + * dto_async_status() gives its status code. Redo the operation on the CPU. + * After a page fault (status 0x03, see DTO_SUBMIT_BOF) the device has done + * dto_async_bytes_completed() bytes; for a move whose buffers overlap those + * bytes have already overwritten part of the source, so finish the move from + * where the device stopped rather than redoing it: the first bytes were done + * when bit 0 of dto_async_result() is 0 (ascending copy), the last when it is + * 1 (descending). + * + * Flags (every submit takes a combination of these): + * + * DTO_SUBMIT_CC directs the operation's output toward the CPU cache + * (IDXD_OP_FLAG_CC) when the device supports it, for destinations that will + * be read again soon. Ignored for operations without a destination (CRC + * generation, compare, translation fetch). + * + * DTO_SUBMIT_BOF makes the device block on a page fault until the kernel has + * resolved it, so the operation completes even if its buffers are not + * faulted in. It is passed to the device as given, and the work queue must + * be configured with block-on-fault ("accel-config config-wq + * --block-on-fault=1"): + * + * WQ block-on-fault DTO_SUBMIT_BOF page fault -> + * enabled set the device waits; the op completes + * disabled set (always) DTO_ASYNC_FAILED, status 0x11 + * (invalid flags): the device rejects it + * either not set DTO_ASYNC_FAILED, status 0x03: partial + * completion, see above + * + * The same applies to each operation of a batch, which dto_batch_poll then + * redoes on the CPU. + * + * CRC values: dto_async_crc_val() and dto_batch_crc() return the raw CRC32C + * of the data -- the SSE4.2 crc32 instruction applied from 0, with no final + * inversion. Standard CRC32C (init 0xFFFFFFFF, final inversion) and chained + * checksums are the seeded "crc32c" calls below, in the iSCSI presentation: + * seed 0 gives the standard CRC32C, and passing one call's result as the next + * call's seed continues the checksum across buffers. + ******************************************************************************/ + +#ifndef DTO_H +#define DTO_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Caller-allocated operation state (opaque). */ +typedef struct dto_async_op { + unsigned char opaque[192] __attribute__((aligned(64))); +} dto_async_op; + +/* Submission flags */ +#define DTO_SUBMIT_CC (1u << 0) /* see above */ +#define DTO_SUBMIT_BOF (1u << 1) /* block on page faults (see above) */ + +/* Submit results */ +#define DTO_ASYNC_SUBMITTED 0 +#define DTO_ASYNC_FALLBACK (-1) + +/* Poll results */ +#define DTO_ASYNC_PENDING 0 +#define DTO_ASYNC_DONE 1 +#define DTO_ASYNC_FAILED (-1) + +/* ---- Data movement ---- */ + +/* Copy n bytes from src to dest. Overlapping buffers are handled like + * memmove. */ +int dto_submit_memcpy(dto_async_op *op, void *dest, const void *src, + size_t n, unsigned int flags); + +/* Fill n bytes of dest with the byte value c (like memset). */ +int dto_submit_memset(dto_async_op *op, void *dest, int c, size_t n, + unsigned int flags); + +/* Fill n bytes of dest with the 8-byte pattern, repeated (the low byte of + * pattern lands at dest[0]). */ +int dto_submit_memfill(dto_async_op *op, void *dest, uint64_t pattern, + size_t n, unsigned int flags); + +/* ---- CRC32C ---- */ + +/* Copy n bytes from src to dest and compute the raw CRC32C of the data + * (read it with dto_async_crc_val). */ +int dto_submit_memcpy_crc(dto_async_op *op, void *dest, const void *src, + size_t n, unsigned int flags); + +/* Raw CRC32C of n bytes at src, without copying (dto_async_crc_val). */ +int dto_submit_crc(dto_async_op *op, const void *src, size_t n, + unsigned int flags); + +/* Seeded CRC32C in the iSCSI presentation (see above), with and without a + * copy. Read the value with dto_async_crc32c_val. */ +int dto_submit_memcpy_crc32c(dto_async_op *op, void *dest, const void *src, + size_t n, uint32_t seed, unsigned int flags); +int dto_submit_crc32c(dto_async_op *op, const void *src, size_t n, + uint32_t seed, unsigned int flags); + +/* Synchronous seeded CRC32C, same presentation as dto_submit_crc32c. Always + * returns the correct value: buffers smaller than DTO_CRC_MIN_BYTES (by + * default DTO_MIN_BYTES) are checksummed on the CPU, as is anything the + * device cannot do. */ +uint32_t dto_crc32c_with_seed(uint32_t seed, const void *src, size_t n); + +/* ---- Compare and translation fetch ---- */ + +/* Compare n bytes at src1 with src2. Once DONE, dto_async_result() is 0 + * when they are equal and 1 when they differ, and dto_async_bytes_completed() + * is then the offset of the first difference. */ +int dto_submit_compare(dto_async_op *op, const void *src1, const void *src2, + size_t n, unsigned int flags); + +/* Compare n bytes at src with the 8-byte pattern, repeated; results as for + * dto_submit_compare. With the buffer's own first 8 bytes as the pattern it + * tells whether the buffer is constant-filled (all zero when they are 0). */ +int dto_submit_compval(dto_async_op *op, const void *src, uint64_t pattern, + size_t n, unsigned int flags); + +/* Translation fetch (DSA 2.0): warm the device's address translations for + * the n bytes at addr without moving data. Devices without the operation + * complete with DTO_ASYNC_FAILED and dto_async_status() == 0x10. */ +int dto_submit_transl_fetch(dto_async_op *op, const void *addr, size_t n, + unsigned int flags); + +/* ---- Completion ---- */ + +/* Poll a submitted operation. Returns DTO_ASYNC_PENDING, DTO_ASYNC_DONE or + * DTO_ASYNC_FAILED. May be called repeatedly, from any thread; once an op + * has left PENDING its result is stable. */ +int dto_async_poll(dto_async_op *op); + +/* Wait until the device has finished with a submitted op, using the wait + * method DTO_WAIT_METHOD selects. It only waits: call dto_async_poll + * afterwards for the result. A no-op for an op that was never submitted or + * whose submit returned DTO_ASYNC_FALLBACK, so it is always safe to call + * before a poll loop. There is deliberately no timeout: until the device + * finishes it still owns the buffers, so the only safe thing to do after a + * timeout would be to wait again. */ +void dto_async_wait(dto_async_op *op); + +/* Nonzero when the configured wait method gives the core back to the + * scheduler while waiting, rather than spinning or pausing on it. For callers + * with their own wait loop deciding whether to delegate to dto_async_wait. + * None of the current wait methods does, so this returns 0. */ +int dto_wait_blocks(void); + +/* Results of a completed (DTO_ASYNC_DONE) operation. */ +uint32_t dto_async_crc_val(const dto_async_op *op); /* raw CRC32C */ +uint32_t dto_async_crc32c_val(const dto_async_op *op); /* seeded (iSCSI) */ +int dto_async_result(const dto_async_op *op); /* compare: 0 equal, 1 differ */ +uint32_t dto_async_bytes_completed(const dto_async_op *op); +/* The device's status code (1 = success, 0 = still pending). */ +int dto_async_status(const dto_async_op *op); + +/* ---- Batches ---- + * + * Up to `capacity` (at most DTO_BATCH_MAX) operations sent to the device as + * one batch descriptor: one submission and one completion to poll instead of + * one per operation. Add operations, submit, then poll; dto_batch_reset makes + * the batch reusable. A batch with a single operation is sent as a plain + * descriptor; an empty one completes immediately. The operations of a batch + * may run in any order and concurrently, so none may depend on another. + * + * dto_batch_add_* return the operation's index, or -1 when the batch is full, + * already submitted, or n is 0 (dualcast also requires bits 11:0 of the two + * destinations to be equal, a device rule). dto_batch_submit returns + * DTO_ASYNC_SUBMITTED, or DTO_ASYNC_FALLBACK with nothing submitted (the + * caller does the operations on the CPU), including when the batch is larger + * than the work queue's max_batch_size or an operation exceeds its maximum + * transfer size. + * + * dto_batch_poll returns DTO_ASYNC_PENDING, then DTO_ASYNC_DONE once every + * operation is complete: operations the device did not finish are redone on + * the CPU before DONE is returned, results included, so after DONE every + * result and CRC is valid. dto_batch_status(b, i) is the device's status for + * operation i (1 when the device did it). A poll of a batch that was never + * submitted, or whose submit fell back, returns DTO_ASYNC_FAILED. + */ +#define DTO_BATCH_MAX 1024 + +typedef struct dto_batch dto_batch; + +/* capacity is clamped to DTO_BATCH_MAX. NULL when out of memory. */ +dto_batch *dto_batch_create(int capacity); +/* Both wait for a batch that is still in flight. */ +void dto_batch_destroy(dto_batch *b); +void dto_batch_reset(dto_batch *b); +int dto_batch_count(const dto_batch *b); + +int dto_batch_add_memmove(dto_batch *b, void *dst, const void *src, size_t n, + unsigned int flags); +int dto_batch_add_memfill(dto_batch *b, void *dst, uint64_t pattern, size_t n, + unsigned int flags); +/* Copy src to both dst1 and dst2. */ +int dto_batch_add_dualcast(dto_batch *b, void *dst1, void *dst2, + const void *src, size_t n, unsigned int flags); +/* Raw CRC32C, read with dto_batch_crc. */ +int dto_batch_add_crc(dto_batch *b, const void *src, size_t n, + unsigned int flags); +int dto_batch_add_memcpy_crc(dto_batch *b, void *dst, const void *src, + size_t n, unsigned int flags); +/* Results as for dto_submit_compare / dto_submit_compval. */ +int dto_batch_add_compare(dto_batch *b, const void *src1, const void *src2, + size_t n, unsigned int flags); +int dto_batch_add_compval(dto_batch *b, const void *src, uint64_t pattern, + size_t n, unsigned int flags); + +int dto_batch_submit(dto_batch *b); +int dto_batch_poll(dto_batch *b); +/* Same contract as dto_async_wait: waits only, poll afterwards. */ +void dto_batch_wait(dto_batch *b); + +/* Results of operation i, valid after dto_batch_poll returned DONE. */ +int dto_batch_status(const dto_batch *b, int i); +int dto_batch_result(const dto_batch *b, int i); +uint32_t dto_batch_bytes_completed(const dto_batch *b, int i); +uint32_t dto_batch_crc(const dto_batch *b, int i); + +/* Convenience: reset b and submit count copies (dst[i] <- src[i], sizes[i] + * bytes) as one batch. Returns as dto_batch_submit; DTO_ASYNC_FALLBACK also + * when count is not between 1 and the batch's capacity or a size is 0. */ +int dto_submit_batch_copy(dto_batch *b, void **dst, void **src, + size_t *sizes, int count, unsigned int flags); + +#ifdef __cplusplus +} +#endif + +#endif /* DTO_H */ diff --git a/tests/test_functional.c b/tests/test_functional.c index f07affe..c49aaba 100644 --- a/tests/test_functional.c +++ b/tests/test_functional.c @@ -19,8 +19,11 @@ ******************************************************************************/ #include "dto_test_utils.h" +#include "../dto.h" +#include #include #include +#include /* Buffer sizes spanning below and above the DSA threshold (65536) */ static const size_t test_sizes[] = { @@ -507,9 +510,800 @@ static int test_multithread(void) /* ---- Test runner ---- */ +/* ---- Explicit asynchronous API (dto.h) ---- + * + * Without DSA every submit falls back and the tests check the documented + * fallback paths. With DTO_TEST_EXPECT_DSA set (a usable work queue + * configured), operations on faulted-in buffers must be submitted and must + * complete on the device, so a regression that quietly pushes work back onto + * the CPU fails instead of passing. + */ + +static int expect_dsa; + +/* CRC32C loop over the SSE4.2 instruction: crc32c_loop(0, ...) is the raw + * presentation and ~crc32c_loop(~seed, ...) the seeded (iSCSI) one. */ +__attribute__((target("sse4.2"))) +static uint32_t crc32c_loop(uint32_t crc, const void *buf, size_t n) +{ + const uint8_t *p = buf; + + for (size_t i = 0; i < n; i++) + crc = _mm_crc32_u8(crc, p[i]); + return crc; +} + +static uint32_t ref_crc_raw(const void *buf, size_t n) +{ + return crc32c_loop(0, buf, n); +} + +static uint32_t ref_crc32c(uint32_t seed, const void *buf, size_t n) +{ + return ~crc32c_loop(~seed, buf, n); +} + +/* Poll until poll_expr leaves PENDING and yield its result. A submitted op + * owns its (stack-allocated) state until it completes, so a stall past the + * 10s safety timeout aborts the process rather than returning into a test + * that would let the device write into a dead stack frame. */ +#define ASYNC_WAIT(poll_expr) ({ \ + struct timespec _start, _now; \ + int _rc; \ + clock_gettime(CLOCK_MONOTONIC, &_start); \ + while ((_rc = (poll_expr)) == DTO_ASYNC_PENDING) { \ + clock_gettime(CLOCK_MONOTONIC, &_now); \ + if (_now.tv_sec - _start.tv_sec > 10) { \ + fprintf(stderr, " FATAL: async op still pending after 10s at %s:%d\n", \ + __FILE__, __LINE__); \ + abort(); \ + } \ + } \ + _rc; }) + +/* The outcome of a submit, then of its poll: SUBMITTED and DONE when DSA is + * expected, any documented outcome otherwise. */ +#define CHECK_SUBMIT(rc) do { \ + if (expect_dsa) \ + ASSERT_EQ((rc), DTO_ASYNC_SUBMITTED); \ + else \ + ASSERT_TRUE((rc) == DTO_ASYNC_SUBMITTED || (rc) == DTO_ASYNC_FALLBACK); \ +} while (0) + +#define CHECK_DONE(rc) do { \ + if (expect_dsa) \ + ASSERT_EQ((rc), DTO_ASYNC_DONE); \ + else \ + ASSERT_TRUE((rc) == DTO_ASYNC_DONE || (rc) == DTO_ASYNC_FAILED); \ +} while (0) + +/* Wait for a submitted op the documented way: wait, then poll. */ +static int finish(dto_async_op *op) +{ + dto_async_wait(op); + return ASYNC_WAIT(dto_async_poll(op)); +} + +/* Submit a memcpy with the given flags and verify the data afterwards, + * whichever of the three documented outcomes (DONE, FAILED + CPU redo, + * FALLBACK + CPU copy) occurs. */ +static int check_async_memcpy(unsigned int flags) +{ + const size_t n = 1 << 20; + uint8_t *src = malloc(n), *dst = malloc(n); + dto_async_op op; + int rc; + + ASSERT_TRUE(src && dst); + fill_pattern(src, n); + clear_buf(dst, n); + + rc = dto_submit_memcpy(&op, dst, src, n, flags); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = finish(&op); + CHECK_DONE(rc); + if (rc != DTO_ASYNC_DONE) + memcpy(dst, src, n); /* documented caller fallback */ + } else { + memcpy(dst, src, n); + } + ASSERT_TRUE(verify_equal(dst, src, n)); + free(src); + free(dst); + return 0; +} + +/* The general tests use faulted-in buffers and no DTO_SUBMIT_BOF, so they + * complete on any work queue; test_async_bof covers block-on-fault. */ +static int test_async_memcpy(void) +{ + return check_async_memcpy(DTO_SUBMIT_CC); +} + +static int test_async_memcpy_no_cc(void) +{ + return check_async_memcpy(0); +} + +static int test_async_memset(void) +{ + const size_t n = (1 << 20) + 3; + uint8_t *dst = malloc(n); + dto_async_op op; + int rc; + + ASSERT_TRUE(dst != NULL); + clear_buf(dst, n); + + rc = dto_submit_memset(&op, dst, 0x5A, n, DTO_SUBMIT_CC); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = finish(&op); + CHECK_DONE(rc); + } + if (rc != DTO_ASYNC_DONE) + memset(dst, 0x5A, n); + ASSERT_TRUE(verify_set(dst, 0x5A, n)); + free(dst); + return 0; +} + +static int test_async_memfill(void) +{ + const uint64_t pattern = 0x0123456789ABCDEFULL; + const size_t n = (256 * 1024) + 5; /* not a multiple of 8 */ + uint8_t *dst = malloc(n); + dto_async_op op; + int rc; + + ASSERT_TRUE(dst != NULL); + clear_buf(dst, n); + + rc = dto_submit_memfill(&op, dst, pattern, n, DTO_SUBMIT_CC); + CHECK_SUBMIT(rc); + if (rc != DTO_ASYNC_SUBMITTED) + return 0; + rc = finish(&op); + CHECK_DONE(rc); + if (rc == DTO_ASYNC_DONE) { + for (size_t i = 0; i < n; i++) + ASSERT_EQ(dst[i], (uint8_t)(pattern >> (8 * (i & 7)))); + } + free(dst); + return 0; +} + +static int test_async_crc(void) +{ + const size_t n = 256 * 1024 + 3; + uint8_t *src = malloc(n); + dto_async_op op; + int rc; + + ASSERT_TRUE(src != NULL); + fill_pattern(src, n); + + rc = dto_submit_crc(&op, src, n, 0); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = finish(&op); + CHECK_DONE(rc); + if (rc == DTO_ASYNC_DONE) + ASSERT_EQ(dto_async_crc_val(&op), ref_crc_raw(src, n)); + } + free(src); + return 0; +} + +static int test_async_memcpy_crc(void) +{ + const size_t n = 256 * 1024; + uint8_t *src = malloc(n), *dst = malloc(n); + dto_async_op op; + int rc; + + ASSERT_TRUE(src && dst); + fill_pattern(src, n); + clear_buf(dst, n); + + rc = dto_submit_memcpy_crc(&op, dst, src, n, DTO_SUBMIT_CC); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = finish(&op); + CHECK_DONE(rc); + } + if (rc == DTO_ASYNC_DONE) + ASSERT_EQ(dto_async_crc_val(&op), ref_crc_raw(src, n)); + else + memcpy(dst, src, n); + ASSERT_TRUE(verify_equal(dst, src, n)); + free(src); + free(dst); + return 0; +} + +/* Seeded (iSCSI) CRC32C: seed 0 is the standard CRC32C, and chaining two + * halves gives the CRC of the whole. */ +static int test_async_crc32c(void) +{ + const size_t n = 128 * 1024, half = n / 2; + uint8_t *src = malloc(n), *dst = malloc(n); + uint32_t first; + dto_async_op op; + int rc; + + ASSERT_TRUE(src && dst); + fill_pattern(src, n); + clear_buf(dst, n); + + /* the reference itself: "123456789" is the CRC32C check string */ + ASSERT_EQ(ref_crc32c(0, "123456789", 9), 0xE3069283u); + + rc = dto_submit_crc32c(&op, src, half, 0, 0); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = finish(&op); + CHECK_DONE(rc); + if (rc == DTO_ASYNC_DONE) { + first = dto_async_crc32c_val(&op); + ASSERT_EQ(first, ref_crc32c(0, src, half)); + rc = dto_submit_memcpy_crc32c(&op, dst + half, src + half, + n - half, first, DTO_SUBMIT_CC); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED && finish(&op) == DTO_ASYNC_DONE) { + ASSERT_EQ(dto_async_crc32c_val(&op), + ref_crc32c(0, src, n)); + ASSERT_TRUE(verify_equal(dst + half, src + half, n - half)); + } + } + } + + /* the synchronous call is always right: CPU below DTO_CRC_MIN_BYTES, + * DSA above, and the seed alone for an empty buffer */ + ASSERT_EQ(dto_crc32c_with_seed(0, src, 100), ref_crc32c(0, src, 100)); + ASSERT_EQ(dto_crc32c_with_seed(0, src, n), ref_crc32c(0, src, n)); + ASSERT_EQ(dto_crc32c_with_seed(dto_crc32c_with_seed(0, src, half), + src + half, n - half), + ref_crc32c(0, src, n)); + ASSERT_EQ(dto_crc32c_with_seed(0x12345678u, src, 0), 0x12345678u); + free(src); + free(dst); + return 0; +} + +/* Offset of the first difference, as compare reports it */ +static int check_compare(const uint8_t *a, const uint8_t *b, size_t n, + int expect_result, uint32_t expect_offset) +{ + dto_async_op op; + int rc = dto_submit_compare(&op, a, b, n, 0); + + CHECK_SUBMIT(rc); + if (rc != DTO_ASYNC_SUBMITTED) + return 0; + rc = finish(&op); + CHECK_DONE(rc); + if (rc == DTO_ASYNC_DONE) { + ASSERT_EQ(dto_async_result(&op), expect_result); + if (expect_result) + ASSERT_EQ(dto_async_bytes_completed(&op), expect_offset); + } + return 0; +} + +static int test_async_compare(void) +{ + static const size_t offsets[] = { 0, 1, 7, 8, 63, 4095, 4096, 100003 }; + const size_t n = 128 * 1024; + uint8_t *a = malloc(n), *b = malloc(n); + + ASSERT_TRUE(a && b); + fill_pattern(a, n); + fill_pattern(b, n); + if (check_compare(a, b, n, 0, 0)) + return 1; + for (size_t i = 0; i < sizeof(offsets) / sizeof(offsets[0]); i++) { + b[offsets[i]] ^= 0x40; + if (check_compare(a, b, n, 1, offsets[i])) + return 1; + b[offsets[i]] ^= 0x40; + } + b[n - 1] ^= 1; + if (check_compare(a, b, n, 1, n - 1)) + return 1; + free(a); + free(b); + return 0; +} + +static int test_async_compval(void) +{ + const uint64_t pattern = 0xCCCCCCCC00000000ULL | 0x5A5A0F0F; + const size_t n = 64 * 1024; + uint8_t *buf = malloc(n); + dto_async_op op; + int rc; + + ASSERT_TRUE(buf != NULL); + for (size_t i = 0; i < n; i++) + ((volatile uint8_t *)buf)[i] = (uint8_t)(pattern >> (8 * (i & 7))); + + for (int round = 0; round < 2; round++) { + /* round 0: buf is the pattern; round 1: one byte differs */ + const size_t off = 12345; + + if (round == 1) + buf[off] ^= 0xFF; + rc = dto_submit_compval(&op, buf, pattern, n, 0); + CHECK_SUBMIT(rc); + if (rc != DTO_ASYNC_SUBMITTED) + continue; + rc = finish(&op); + CHECK_DONE(rc); + if (rc != DTO_ASYNC_DONE) + continue; + ASSERT_EQ(dto_async_result(&op), round); + if (round == 1) + ASSERT_EQ(dto_async_bytes_completed(&op), off); + } + free(buf); + return 0; +} + +static int test_async_transl_fetch(void) +{ + const size_t n = 1 << 20; + uint8_t *buf = malloc(n); + dto_async_op op; + int rc; + + ASSERT_TRUE(buf != NULL); + clear_buf(buf, n); + rc = dto_submit_transl_fetch(&op, buf, n, 0); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = finish(&op); + /* devices without the operation report BAD_OPCODE */ + if (rc != DTO_ASYNC_DONE) { + ASSERT_EQ(rc, DTO_ASYNC_FAILED); + ASSERT_EQ(dto_async_status(&op), 0x10); + } + } + free(buf); + return 0; +} + +/* DTO_SUBMIT_BOF is passed to the device as given (dto.h). Learn whether the + * work queue has block-on-fault from a BOF submit on faulted-in memory, then + * copy into a destination whose first pages are touched and the rest not: + * BOF set, WQ with block-on-fault: the device waits; the copy completes + * BOF set, WQ without block-on-fault: rejected, status 0x11 (invalid flags) + * BOF not set, any WQ: stops at the first untouched page, + * status 0x03, the touched part done + * Assumes every work queue DTO uses has the same block-on-fault setting. */ +static int test_async_bof(void) +{ + const size_t page = 4096, n = 16 * page, warm = 4 * page; + uint8_t *src = malloc(n), *hot = malloc(n); + dto_batch *b = dto_batch_create(2); + dto_async_op op; + int rc, wq_bof; + + ASSERT_TRUE(src && hot && b); + fill_pattern(src, n); + clear_buf(hot, n); + + rc = dto_submit_memcpy(&op, hot, src, n, DTO_SUBMIT_BOF); + CHECK_SUBMIT(rc); + if (rc != DTO_ASYNC_SUBMITTED) + return 0; + rc = finish(&op); + if (rc == DTO_ASYNC_DONE) { + wq_bof = 1; + ASSERT_TRUE(verify_equal(hot, src, n)); + } else { + ASSERT_EQ(rc, DTO_ASYNC_FAILED); + ASSERT_EQ(dto_async_status(&op), 0x11); + wq_bof = 0; + } + printf(" (work queue block-on-fault: %s)\n", wq_bof ? "enabled" : "disabled"); + + for (int bof = 0; bof < 2; bof++) { + uint8_t *dst = mmap(NULL, n, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + ASSERT_TRUE(dst != MAP_FAILED); + clear_buf(dst, warm); /* touch the first pages only */ + rc = dto_submit_memcpy(&op, dst, src, n, bof ? DTO_SUBMIT_BOF : 0); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = finish(&op); + if (bof && wq_bof) { + ASSERT_EQ(rc, DTO_ASYNC_DONE); + ASSERT_TRUE(verify_equal(dst, src, n)); + } else if (bof) { + ASSERT_EQ(rc, DTO_ASYNC_FAILED); + ASSERT_EQ(dto_async_status(&op), 0x11); + } else { + ASSERT_EQ(rc, DTO_ASYNC_FAILED); + ASSERT_EQ(dto_async_status(&op), 0x03); + ASSERT_EQ(dto_async_bytes_completed(&op), warm); + ASSERT_TRUE(verify_equal(dst, src, warm)); + } + } + munmap(dst, n); + } + + /* the same per batch operation; dto_batch_poll redoes the rejected ones */ + clear_buf(hot, n); + ASSERT_EQ(dto_batch_add_memmove(b, hot, src, n / 2, DTO_SUBMIT_BOF), 0); + ASSERT_EQ(dto_batch_add_memmove(b, hot + n / 2, src + n / 2, n / 2, 0), 1); + rc = dto_batch_submit(b); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + ASSERT_EQ(ASYNC_WAIT(dto_batch_poll(b)), DTO_ASYNC_DONE); + ASSERT_EQ(dto_batch_status(b, 0), wq_bof ? 1 : 0x11); + ASSERT_EQ(dto_batch_status(b, 1), 1); + ASSERT_TRUE(verify_equal(hot, src, n)); + } + dto_batch_destroy(b); + free(src); + free(hot); + return 0; +} + +/* dto_async_wait must return at once for ops that were never submitted or + * fell back: their status byte will never be written. */ +static int test_async_wait_unsubmitted(void) +{ + dto_async_op op; + uint8_t a[64], b[64]; + + memset(&op, 0xAB, sizeof(op)); + dto_async_wait(&op); + ASSERT_EQ(dto_submit_memcpy(&op, a, b, 0, 0), DTO_ASYNC_FALLBACK); + dto_async_wait(&op); + ASSERT_EQ(dto_wait_blocks(), 0); + return 0; +} + +static int test_async_bad_args(void) +{ + uint8_t a[64], b[64]; + struct { char pad[16]; dto_async_op op; } __attribute__((aligned(64))) s; + dto_async_op *unaligned = (dto_async_op *)&s.pad[8]; + dto_async_op op; + + /* zero length never reaches the device */ + ASSERT_EQ(dto_submit_memcpy(&op, a, b, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_memset(&op, a, 0, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_memfill(&op, a, 0, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_crc(&op, b, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_crc32c(&op, b, 0, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_memcpy_crc(&op, a, b, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_compare(&op, a, b, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_compval(&op, a, 0, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_transl_fetch(&op, a, 0, 0), DTO_ASYNC_FALLBACK); + /* the completion record needs 32-byte alignment */ + ASSERT_EQ(dto_submit_memcpy(unaligned, a, b, sizeof(a), 0), + DTO_ASYNC_FALLBACK); + return 0; +} + +/* ---- Batches ---- */ + +static int test_batch_limits(void) +{ + dto_batch *b = dto_batch_create(DTO_BATCH_MAX + 5); + uint8_t *buf = aligned_alloc(4096, 3 * 4096); + int i; + + ASSERT_TRUE(b && buf); + clear_buf(buf, 3 * 4096); + /* polling a batch that was never submitted reports FAILED */ + ASSERT_EQ(dto_batch_poll(b), DTO_ASYNC_FAILED); + /* an empty batch completes at once */ + ASSERT_EQ(dto_batch_submit(b), DTO_ASYNC_SUBMITTED); + ASSERT_EQ(dto_batch_poll(b), DTO_ASYNC_DONE); + /* a submitted batch takes no more operations until reset */ + ASSERT_EQ(dto_batch_add_memfill(b, buf, 0, 64, 0), -1); + dto_batch_reset(b); + + ASSERT_EQ(dto_batch_add_memmove(b, buf, buf + 64, 0, 0), -1); + /* dualcast destinations must agree in bits 11:0 */ + ASSERT_EQ(dto_batch_add_dualcast(b, buf + 4096, buf + 2 * 4096 + 64, + buf, 64, 0), -1); + /* the capacity is clamped to DTO_BATCH_MAX */ + for (i = 0; i < DTO_BATCH_MAX; i++) + ASSERT_EQ(dto_batch_add_compval(b, buf, 0, 64, 0), i); + ASSERT_EQ(dto_batch_add_compval(b, buf, 0, 64, 0), -1); + ASSERT_EQ(dto_batch_count(b), DTO_BATCH_MAX); + dto_batch_reset(b); + ASSERT_EQ(dto_batch_count(b), 0); + dto_batch_destroy(b); + dto_batch_destroy(NULL); + free(buf); + return 0; +} + +/* Every operation kind in one batch, checked against CPU results. */ +static int run_batch_all_ops(int nops_limit) +{ + const size_t n = 32 * 1024 + 8; + const uint64_t pattern = 0x1122334455667788ULL; + uint8_t *src = aligned_alloc(4096, n + 4096); + uint8_t *src2 = aligned_alloc(4096, n + 4096); + uint8_t *mv = aligned_alloc(4096, n + 4096); + uint8_t *fill = aligned_alloc(4096, n + 4096); + uint8_t *d1 = aligned_alloc(4096, n + 4096); + uint8_t *d2 = aligned_alloc(4096, n + 4096); + uint8_t *cc = aligned_alloc(4096, n + 4096); + uint8_t *pat = aligned_alloc(4096, n + 4096); + dto_batch *b = dto_batch_create(16); + int ops[7], count = 0, rc; + + ASSERT_TRUE(src && src2 && mv && fill && d1 && d2 && cc && pat && b); + /* operations in one batch may run concurrently, so none of them reads + * what another writes */ + for (size_t i = 0; i < n; i++) + ((volatile uint8_t *)pat)[i] = (uint8_t)(pattern >> (8 * (i & 7))); + fill_pattern(src, n); + fill_pattern(src2, n); + src2[777] ^= 1; + clear_buf(mv, n); + clear_buf(fill, n); + clear_buf(d1, n); + clear_buf(d2, n); + clear_buf(cc, n); + + ops[0] = dto_batch_add_memmove(b, mv, src, n, DTO_SUBMIT_CC); + ops[1] = dto_batch_add_memfill(b, fill, pattern, n, 0); + ops[2] = dto_batch_add_dualcast(b, d1, d2, src, n, 0); + ops[3] = dto_batch_add_crc(b, src, n, 0); + ops[4] = dto_batch_add_memcpy_crc(b, cc, src, n, DTO_SUBMIT_CC); + ops[5] = dto_batch_add_compare(b, src, src2, n, 0); + ops[6] = dto_batch_add_compval(b, pat, pattern, n, 0); + for (int i = 0; i < 7; i++) + ASSERT_EQ(ops[i], i); + count = 7; + /* nops_limit 1 exercises the plain-descriptor path */ + if (nops_limit == 1) { + dto_batch_reset(b); + ASSERT_EQ(dto_batch_add_memcpy_crc(b, cc, src, n, 0), 0); + count = 1; + } + + rc = dto_batch_submit(b); + CHECK_SUBMIT(rc); + if (rc != DTO_ASYNC_SUBMITTED) + return 0; + dto_batch_wait(b); + ASSERT_EQ(ASYNC_WAIT(dto_batch_poll(b)), DTO_ASYNC_DONE); + ASSERT_EQ(dto_batch_poll(b), DTO_ASYNC_DONE); /* stays DONE */ + for (int i = 0; i < count; i++) { + if (expect_dsa) + ASSERT_EQ(dto_batch_status(b, i), 1); + } + + if (count == 1) { + ASSERT_TRUE(verify_equal(cc, src, n)); + ASSERT_EQ(dto_batch_crc(b, 0), ref_crc_raw(src, n)); + } else { + ASSERT_TRUE(verify_equal(mv, src, n)); + for (size_t i = 0; i < n; i++) + ASSERT_EQ(fill[i], (uint8_t)(pattern >> (8 * (i & 7)))); + ASSERT_TRUE(verify_equal(d1, src, n)); + ASSERT_TRUE(verify_equal(d2, src, n)); + ASSERT_EQ(dto_batch_crc(b, 3), ref_crc_raw(src, n)); + ASSERT_TRUE(verify_equal(cc, src, n)); + ASSERT_EQ(dto_batch_crc(b, 4), ref_crc_raw(src, n)); + ASSERT_EQ(dto_batch_result(b, 5), 1); + ASSERT_EQ(dto_batch_bytes_completed(b, 5), 777); + ASSERT_EQ(dto_batch_result(b, 6), 0); + } + dto_batch_destroy(b); + free(src); + free(src2); + free(mv); + free(fill); + free(d1); + free(d2); + free(cc); + free(pat); + return 0; +} + +static int test_batch_all_ops(void) +{ + return run_batch_all_ops(0); +} + +static int test_batch_single_op(void) +{ + return run_batch_all_ops(1); +} + +/* Operations the device cannot finish are redone by dto_batch_poll. Fresh + * anonymous pages are not mapped until first touched, so without + * block-on-fault the device stops at them; the results must still be exactly + * what the CPU computes, including for overlapping moves the device left + * half done in either direction. */ +static int test_batch_repair(void) +{ + const size_t page = 4096, region = 16 * page; + const uint64_t pattern = 0xA5A55A5A0F0FF0F0ULL; + uint8_t *m = mmap(NULL, 8 * region, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint8_t *up, *down, *cold, *fill, *cmp, *cc; + uint8_t *exp_up, *exp_down; + const size_t mv = 8 * page; /* overlapping move length */ + dto_batch *b = dto_batch_create(8); + int rc; + + ASSERT_TRUE(m != MAP_FAILED && b); + up = m; /* ascending move: low pages touched */ + down = m + region; /* descending move: high pages touched */ + cold = m + 2 * region; /* never touched: reads as zero */ + fill = m + 3 * region; + cmp = m + 4 * region; + cc = m + 5 * region; + exp_up = malloc(region); + exp_down = malloc(region); + ASSERT_TRUE(exp_up && exp_down); + + /* up: move [4p, 12p) down to [0, 8p); the device copies upward and + * stops at the untouched pages from 10p, after overwriting [4p, 6p) + * of its own source, so redoing the whole move would be wrong. */ + for (size_t i = 0; i < 10 * page; i++) + ((volatile uint8_t *)up)[i] = (uint8_t)(i * 7); + for (size_t i = 0; i < region; i++) + exp_up[i] = i < 10 * page ? (uint8_t)(i * 7) : 0; + for (size_t i = 0; i < mv; i++) /* forward copy is memmove here */ + exp_up[i] = exp_up[4 * page + i]; + + /* down: move [0, 8p) up to [4p, 12p); the device copies downward and + * stops at the untouched pages below 2p, after overwriting [6p, 8p) + * of its own source. */ + for (size_t i = 2 * page; i < region; i++) + ((volatile uint8_t *)down)[i] = (uint8_t)(i * 13); + for (size_t i = 0; i < region; i++) + exp_down[i] = i >= 2 * page ? (uint8_t)(i * 13) : 0; + for (size_t i = mv; i-- > 0;) /* backward copy is memmove here */ + exp_down[4 * page + i] = exp_down[i]; + + ASSERT_TRUE(dto_batch_add_memmove(b, up, up + 4 * page, mv, 0) == 0); + ASSERT_TRUE(dto_batch_add_memmove(b, down + 4 * page, down, mv, 0) == 1); + ASSERT_TRUE(dto_batch_add_memfill(b, fill, pattern, region, 0) == 2); + ASSERT_TRUE(dto_batch_add_crc(b, cold, region, 0) == 3); + ASSERT_TRUE(dto_batch_add_compval(b, cold, 0, region, 0) == 4); + ASSERT_TRUE(dto_batch_add_compare(b, cold, cmp, region, 0) == 5); + ASSERT_TRUE(dto_batch_add_memcpy_crc(b, cc, cold, region, 0) == 6); + + rc = dto_batch_submit(b); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + ASSERT_EQ(ASYNC_WAIT(dto_batch_poll(b)), DTO_ASYNC_DONE); + ASSERT_TRUE(verify_equal(up, exp_up, region)); + ASSERT_TRUE(verify_equal(down, exp_down, region)); + for (size_t i = 0; i < region; i++) + ASSERT_EQ(fill[i], (uint8_t)(pattern >> (8 * (i & 7)))); + ASSERT_EQ(dto_batch_crc(b, 3), ref_crc_raw(cold, region)); + ASSERT_EQ(dto_batch_result(b, 4), 0); + ASSERT_EQ(dto_batch_result(b, 5), 0); + ASSERT_EQ(dto_batch_crc(b, 6), ref_crc_raw(cold, region)); + ASSERT_TRUE(verify_set(cc, 0, region)); + } + dto_batch_destroy(b); + munmap(m, 8 * region); + free(exp_up); + free(exp_down); + return 0; +} + +/* Two threads polling one batch that needs repair: the overlapping move is + * not idempotent, so a repair done twice would show. */ +struct poll_arg { + dto_batch *b; + int rc; +}; + +static void *poller(void *p) +{ + struct poll_arg *a = p; + + a->rc = ASYNC_WAIT(dto_batch_poll(a->b)); + return NULL; +} + +static int test_batch_concurrent_poll(void) +{ + const size_t page = 4096, region = 16 * page, mv = 8 * page; + uint8_t *m = mmap(NULL, region, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint8_t *exp = malloc(region); + dto_batch *b = dto_batch_create(2); + struct poll_arg args[2]; + pthread_t t[2]; + int rc; + + ASSERT_TRUE(m != MAP_FAILED && exp && b); + for (size_t i = 2 * page; i < region; i++) + ((volatile uint8_t *)m)[i] = (uint8_t)(i * 3); + for (size_t i = 0; i < region; i++) + exp[i] = i >= 2 * page ? (uint8_t)(i * 3) : 0; + for (size_t i = mv; i-- > 0;) + exp[4 * page + i] = exp[i]; + + ASSERT_EQ(dto_batch_add_memmove(b, m + 4 * page, m, mv, 0), 0); + rc = dto_batch_submit(b); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + for (int i = 0; i < 2; i++) { + args[i].b = b; + pthread_create(&t[i], NULL, poller, &args[i]); + } + for (int i = 0; i < 2; i++) { + pthread_join(t[i], NULL); + ASSERT_EQ(args[i].rc, DTO_ASYNC_DONE); + } + ASSERT_TRUE(verify_equal(m, exp, region)); + } + dto_batch_destroy(b); + munmap(m, region); + free(exp); + return 0; +} + +static int test_async_batch_copy(void) +{ + enum { COUNT = 8 }; + const size_t base = 16 * 1024; + uint8_t *src[COUNT], *dst[COUNT]; + void *srcv[COUNT], *dstv[COUNT]; + size_t sizes[COUNT]; + dto_batch *b = dto_batch_create(COUNT); + int rc; + + ASSERT_TRUE(b != NULL); + for (int i = 0; i < COUNT; i++) { + sizes[i] = base * (i + 1) + i * 64; /* varied, some unaligned */ + src[i] = malloc(sizes[i]); + dst[i] = malloc(sizes[i]); + ASSERT_TRUE(src[i] && dst[i]); + fill_pattern(src[i], sizes[i]); + clear_buf(dst[i], sizes[i]); + srcv[i] = src[i]; + dstv[i] = dst[i]; + } + + /* more copies than the batch holds cannot be submitted */ + ASSERT_EQ(dto_submit_batch_copy(b, dstv, srcv, sizes, COUNT + 1, 0), + DTO_ASYNC_FALLBACK); + rc = dto_submit_batch_copy(b, dstv, srcv, sizes, COUNT, DTO_SUBMIT_CC); + CHECK_SUBMIT(rc); + if (rc == DTO_ASYNC_SUBMITTED) { + dto_batch_wait(b); + ASSERT_EQ(ASYNC_WAIT(dto_batch_poll(b)), DTO_ASYNC_DONE); + } else { + for (int i = 0; i < COUNT; i++) + memcpy(dst[i], src[i], sizes[i]); + } + for (int i = 0; i < COUNT; i++) { + ASSERT_TRUE(verify_equal(dst[i], src[i], sizes[i])); + free(src[i]); + free(dst[i]); + } + dto_batch_destroy(b); + return 0; +} + int main(void) { - printf("DTO Functional Tests\n"); + expect_dsa = getenv("DTO_TEST_EXPECT_DSA") != NULL; + + printf("DTO Functional Tests%s\n", + expect_dsa ? " (DTO_TEST_EXPECT_DSA: offload required)" : ""); printf("====================\n\n"); struct dto_test tests[] = { @@ -523,6 +1317,25 @@ int main(void) TEST_ENTRY(test_zero_length), TEST_ENTRY(test_memcpy_unaligned), TEST_ENTRY(test_multithread), + TEST_ENTRY(test_async_memcpy), + TEST_ENTRY(test_async_memcpy_no_cc), + TEST_ENTRY(test_async_memset), + TEST_ENTRY(test_async_memfill), + TEST_ENTRY(test_async_crc), + TEST_ENTRY(test_async_memcpy_crc), + TEST_ENTRY(test_async_crc32c), + TEST_ENTRY(test_async_compare), + TEST_ENTRY(test_async_compval), + TEST_ENTRY(test_async_transl_fetch), + TEST_ENTRY(test_async_bof), + TEST_ENTRY(test_async_wait_unsubmitted), + TEST_ENTRY(test_async_bad_args), + TEST_ENTRY(test_batch_limits), + TEST_ENTRY(test_batch_all_ops), + TEST_ENTRY(test_batch_single_op), + TEST_ENTRY(test_batch_repair), + TEST_ENTRY(test_batch_concurrent_poll), + TEST_ENTRY(test_async_batch_copy), {NULL, NULL} };