Add explicit asynchronous submit/poll offload API

CMakeLists.txt

@@ -23,18 +23,34 @@ option(DTO_NUMA_SUPPORT "Enable NUMA awareness support (requires libnuma)" ON)
 option(DTO_STATIC_ACCEL_CONFIG "Statically link accel-config into libdto.so" OFF)
 option(DTO_STATIC_NUMA "Statically link numa into libdto.so (requires PIC-compiled libnuma.a)" OFF)
 
+# libdto_explicit is the same library without the memcpy/memset/memmove/
+# memcmp interposers, for programs that call the dto_* API (dto.h) directly
+# and must not have every large libc copy in the process sent to DSA.
+option(DTO_BUILD_EXPLICIT "Also build libdto_explicit, without libc interposition" ON)
+
 # Build the shared library
 add_library(dto SHARED dto.c)
 add_library(DTO::dto ALIAS dto)
-target_include_directories(dto
-    PUBLIC
-        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-        $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
+set(DTO_LIBRARIES dto)
+if(DTO_BUILD_EXPLICIT)
+    add_library(dto_explicit SHARED dto.c)
+    add_library(DTO::dto_explicit ALIAS dto_explicit)
+    target_compile_definitions(dto_explicit PRIVATE DTO_NO_LIBC_INTERPOSE)
+    list(APPEND DTO_LIBRARIES dto_explicit)
+endif()
+
+foreach(lib ${DTO_LIBRARIES})
+    target_include_directories(${lib}
+        PUBLIC
+            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+            $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
+endforeach()
 
 # set gnu source everywhere
 add_compile_definitions(_GNU_SOURCE)
 
-# Add the -DDTO_STATS_SUPPORT preprocessor definition
+# Add the -DDTO_STATS_SUPPORT preprocessor definition. Stats cover the
+# interposed calls, so libdto_explicit has none to collect.
 target_compile_definitions(dto PRIVATE DTO_STATS_SUPPORT)
 
 # Link libraries
@@ -43,7 +59,9 @@ set(DTO_DYNAMIC_LIBS dl)
 
 # Handle accel-config support
 if(DTO_ACCEL_CONFIG_SUPPORT)
-    target_compile_definitions(dto PRIVATE DTO_ACCEL_CONFIG_SUPPORT)
+    foreach(lib ${DTO_LIBRARIES})
+        target_compile_definitions(${lib} PRIVATE DTO_ACCEL_CONFIG_SUPPORT)
+    endforeach()
     message(STATUS "accel-config support: enabled")
 
     if(DTO_STATIC_ACCEL_CONFIG)
@@ -62,7 +80,9 @@ endif()
 
 # Handle numa support
 if(DTO_NUMA_SUPPORT)
-    target_compile_definitions(dto PRIVATE DTO_NUMA_SUPPORT)
+    foreach(lib ${DTO_LIBRARIES})
+        target_compile_definitions(${lib} PRIVATE DTO_NUMA_SUPPORT)
+    endforeach()
     message(STATUS "NUMA support: enabled")
 
     if(DTO_STATIC_NUMA)
@@ -83,21 +103,25 @@ endif()
 # Link everything together. Use PRIVATE so accel-config/numa/dl are the shared
 # library's own dependencies (resolved via its DT_NEEDED) rather than INTERFACE
 # requirements that would be forced onto every downstream consumer of DTO::dto.
-if(DTO_STATIC_LIBS)
-    target_link_libraries(dto PRIVATE
-        -Wl,--whole-archive
-        ${DTO_STATIC_LIBS}
-        -Wl,--no-whole-archive
-        ${DTO_DYNAMIC_LIBS}
-    )
-else()
-    target_link_libraries(dto PRIVATE ${DTO_DYNAMIC_LIBS})
-endif()
+foreach(lib ${DTO_LIBRARIES})
+    if(DTO_STATIC_LIBS)
+        target_link_libraries(${lib} PRIVATE
+            -Wl,--whole-archive
+            ${DTO_STATIC_LIBS}
+            -Wl,--no-whole-archive
+            ${DTO_DYNAMIC_LIBS}
+        )
+    else()
+        target_link_libraries(${lib} PRIVATE ${DTO_DYNAMIC_LIBS})
+    endif()
+endforeach()
 
 include(CheckCCompilerFlag)
 check_c_compiler_flag("-mwaitpkg" HAS_WAITPKG)
 if (HAS_WAITPKG)
-  target_compile_options(dto PRIVATE -mwaitpkg -march=native)
+  foreach(lib ${DTO_LIBRARIES})
+    target_compile_options(${lib} PRIVATE -mwaitpkg -march=native)
+  endforeach()
 endif()
 
 # Build dto-test and dto-test-wodto
@@ -123,6 +147,18 @@ if(DTO_BUILD_TESTS)
         TIMEOUT 120
     )
 
+    # The same tests against libdto_explicit: the explicit API without the
+    # interposers (the libc mem* tests then exercise libc itself).
+    if(DTO_BUILD_EXPLICIT)
+        add_executable(dto-test-functional-explicit tests/test_functional.c)
+        target_link_libraries(dto-test-functional-explicit PRIVATE DTO::dto_explicit pthread)
+        add_test(NAME functional_explicit COMMAND dto-test-functional-explicit)
+        set_tests_properties(functional_explicit PROPERTIES
+            LABELS "functional"
+            TIMEOUT 120
+        )
+    endif()
+
     # ---- A/B performance regression test ----
     # The baseline object is compiled from unguarded upstream sources that
     # always reference accel-config and numa, so the perf test can only be
@@ -225,8 +261,10 @@ if(DTO_BUILD_TESTS)
     endif()
 endif()
 
-# Install and export the library
-install(TARGETS dto
+install(FILES dto.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+
+# Install and export the libraries
+install(TARGETS ${DTO_LIBRARIES}
         EXPORT DTOTargets
         RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
         LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}

Makefile

@@ -12,8 +12,13 @@ libdto: dto.c
 libdto_nostats: dto.c
 	gcc -shared -fPIC -Wl,-soname,libdto.so dto.c $(DML_LIB_CXX) -DDTO_ACCEL_CONFIG_SUPPORT -DDTO_NUMA_SUPPORT -o libdto.so.1.0 -laccel-config -ldl -lnuma -mwaitpkg
 
+# the explicit dto_* API without the libc memcpy/memset/memmove/memcmp interposers
+libdto_explicit: dto.c
+	gcc -shared -fPIC -Wl,-soname,libdto_explicit.so dto.c $(DML_LIB_CXX) -DDTO_NO_LIBC_INTERPOSE -DDTO_ACCEL_CONFIG_SUPPORT -DDTO_NUMA_SUPPORT -o libdto_explicit.so.1.0 -laccel-config -ldl -lnuma -mwaitpkg
+
 install:
 	cp libdto.so.1.0 /usr/lib64/
+	cp dto.h /usr/include/
 	ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so.1
 	ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so
 

README.md

@@ -71,6 +71,7 @@ Following environment variables control the behavior of DTO library:
    DTO_UMWAIT_DELAY=xxxx defines delay for umwait command (check max possible value at: /sys/devices/system/cpu/umwait_control/max_time), default is 100000
 	DTO_LOG_FILE=<dto log file path> Redirect the DTO output to the specified file instead of std output (useful for debugging and statistics collection). file name is suffixed by process pid.
 	DTO_LOG_LEVEL=0/1/2 controls the log level. higher value means more verbose logging (default 0).
+	DTO_CRC_MIN_BYTES=xxxx smallest buffer dto_crc32c_with_seed() sends to DSA; smaller ones are checksummed on the CPU (default: DTO_MIN_BYTES).
 ```
 
 Although not the only usage models of DTO, the following are some common ones:
@@ -95,6 +96,51 @@ Although not the only usage models of DTO, the following are some common ones:
       DTO_WAIT_METHOD=yield or umwait (saves either cycles or power)
 
 
+## Explicit asynchronous API
+
+Besides intercepting the libc calls, libdto exports an explicit submit/poll
+API, declared in `dto.h`, for applications that want to overlap DSA work with
+their own: copy, fill, CRC32C (optionally fused with a copy), compare, compare
+against a pattern and translation fetch, each as a single operation or batched
+(up to 1024 operations in one DSA batch descriptor).
+
+```c
+#include <dto.h>
+
+dto_async_op op;                    /* caller-owned, 64-byte aligned */
+
+if (dto_submit_memcpy_crc(&op, dst, src, n, DTO_SUBMIT_CC) ==
+    DTO_ASYNC_SUBMITTED) {
+	do_other_work();
+	dto_async_wait(&op);            /* waits with DTO_WAIT_METHOD */
+	if (dto_async_poll(&op) == DTO_ASYNC_DONE)
+		crc = dto_async_crc_val(&op);
+	else
+		redo_on_cpu();
+} else {                            /* DTO_ASYNC_FALLBACK: nothing was done */
+	redo_on_cpu();
+}
+```
+
+A batch is built with `dto_batch_create`, `dto_batch_add_*` and
+`dto_batch_submit`; `dto_batch_poll` redoes on the CPU anything the device
+could not finish, so once it returns `DTO_ASYNC_DONE` every result is valid.
+
+Explicit submits are not subject to `DTO_MIN_BYTES`: the caller decides what to
+offload. `DTO_SUBMIT_BOF` makes the device wait for page faults to be
+resolved; it requires a work queue configured with block-on-fault
+(`accel-config config-wq --block-on-fault=1`), and on one without it the
+device rejects the operation. Without the flag a page fault stops the
+operation partway, on any work queue. `dto_async_crc_val` and `dto_batch_crc`
+return the raw CRC32C (the SSE4.2 `crc32` instruction applied from 0, no final
+inversion); the seeded `*_crc32c` calls return standard CRC32C and chain across
+buffers. See `dto.h` for the full contract.
+
+Programs that only use this API, and must not have their own large libc copies
+sent to DSA, can link `libdto_explicit` instead: the same library without the
+memcpy/memset/memmove/memcmp interposers (`-ldto_explicit`, or
+`DTO::dto_explicit` from CMake).
+
 ## Build
 
 Pre-requisite packages:
@@ -124,6 +170,7 @@ The accel-config and libnuma dependencies are optional and can be disabled at co
 | `DTO_NUMA_SUPPORT` | `ON` | Use libnuma for NUMA-aware work queue selection. When disabled, libdto does not link against libnuma and the `DTO_IS_NUMA_AWARE` environment variable has no effect (work queues are used in round-robin order). |
 | `DTO_STATIC_ACCEL_CONFIG` | `OFF` | Statically link libaccel-config into libdto.so instead of depending on the shared library at runtime. Requires `libaccel-config.a` to be installed. |
 | `DTO_STATIC_NUMA` | `OFF` | Statically link libnuma into libdto.so instead of depending on the shared library at runtime. Requires a `libnuma.a` compiled with `-fPIC`. |
+| `DTO_BUILD_EXPLICIT` | `ON` | Also build `libdto_explicit`, the explicit API (`dto.h`) without the libc interposers. |
 
 Example: build without either dependency:
 ```bash

TESTING.md

@@ -59,6 +59,26 @@ hardware -- DTO falls back to CPU transparently.
 ctest -R functional
 ```
 
+They also cover the explicit asynchronous API (`dto.h`): every submit and
+batch operation is checked against the CPU result, including the CPU repair of
+batch operations the device could not finish. `functional_explicit` runs the
+same tests against `libdto_explicit`.
+
+Because every explicit operation may legitimately fall back to the CPU, a
+broken offload path would still pass. On a machine with a usable work queue,
+set `DTO_TEST_EXPECT_DSA` to require that operations on faulted-in buffers are
+submitted and completed by the device:
+
+```bash
+sudo DTO_TEST_EXPECT_DSA=1 ./dto-test-functional
+sudo DTO_TEST_EXPECT_DSA=1 ./dto-test-functional-explicit
+```
+
+`test_async_bof` checks the block-on-fault behavior against whichever setting
+the work queue has (it reports which), so run the suite once with
+`block_on_fault` enabled and once without to cover both. It assumes all work
+queues DTO uses share the same setting.
+
 ## Performance Tests
 
 The performance test (`perf`) detects regressions in libdto by