diff --git a/.github/workflows/Engine-CI.yml b/.github/workflows/Engine-CI.yml index 0ac1bff02..fb2307c02 100644 --- a/.github/workflows/Engine-CI.yml +++ b/.github/workflows/Engine-CI.yml @@ -9,6 +9,7 @@ on: - 'Obelisk/**' - 'Resources/**' - 'Scripts/**' + - 'tests/**' - 'CMakeLists.txt' - 'CMakePresets.json' - 'dependencies.cmake' @@ -21,6 +22,7 @@ on: - 'Obelisk/**' - 'Resources/**' - 'Scripts/**' + - 'tests/**' - 'CMakeLists.txt' - 'CMakePresets.json' - 'dependencies.cmake' @@ -41,25 +43,11 @@ on: default: false jobs: + scene-sanitizers: + uses: ./.github/workflows/job-scene-sanitizers.yml + scene-evidence-tests: - name: Scene evidence tests (${{ matrix.os }}) - runs-on: ${{ matrix.os }} - timeout-minutes: 10 - permissions: - contents: read - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04, windows-2025, macos-latest] - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Validate scene lifecycle and capture evidence tools - run: python -m unittest discover -s Scripts/tests -v + uses: ./.github/workflows/job-scene-evidence-tests.yml commitlint: # A develop-to-main promotion contains historical commits that predate diff --git a/.github/workflows/job-scene-evidence-tests.yml b/.github/workflows/job-scene-evidence-tests.yml new file mode 100644 index 000000000..b15218fd9 --- /dev/null +++ b/.github/workflows/job-scene-evidence-tests.yml @@ -0,0 +1,25 @@ +name: Scene evidence tests + +on: + workflow_call: + +jobs: + scene-evidence-tests: + name: Scene evidence tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, windows-2025, macos-latest] + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Validate scene lifecycle and capture evidence tools + run: python -m unittest discover -s tests/scene/unit -v diff --git a/.github/workflows/job-scene-sanitizers.yml b/.github/workflows/job-scene-sanitizers.yml new file mode 100644 index 000000000..5ce3d3065 --- /dev/null +++ b/.github/workflows/job-scene-sanitizers.yml @@ -0,0 +1,50 @@ +name: Scene CPU sanitizers + +on: + workflow_call: + +jobs: + scene-sanitizers: + name: Scene CPU sanitizer (${{ matrix.name }}) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: + - name: address-undefined + sanitizer: address,undefined + - name: thread + sanitizer: thread + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y ninja-build cmake gcc g++ libasound2-dev libgl1-mesa-dev libpulse-dev libdbus-1-dev libglib2.0-dev libx11-dev libxcursor-dev libxext-dev libxi-dev libxinerama-dev libxrandr-dev libxss-dev libxt-dev libxxf86vm-dev glslang-tools libwayland-dev libxkbcommon-dev pkg-config + - name: Configure instrumented build + env: + CC: gcc + CXX: g++ + SANITIZER: ${{ matrix.sanitizer }} + run: | + cmake --preset Linux_x64_Debug \ + "-DCMAKE_C_FLAGS=-fsanitize=${SANITIZER} -fno-omit-frame-pointer" \ + "-DCMAKE_CXX_FLAGS=-fsanitize=${SANITIZER} -fno-sanitize=vptr -fno-omit-frame-pointer" \ + "-DCMAKE_EXE_LINKER_FLAGS=-fsanitize=${SANITIZER}" \ + "-DCMAKE_SHARED_LINKER_FLAGS=-fsanitize=${SANITIZER}" + - name: Build scene tests + run: cmake --build Result.Linux.x64.Debug --target ZEngineTests -j 2 + - name: Run lifecycle and resource tests + env: + ASAN_OPTIONS: detect_leaks=1:halt_on_error=1 + UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1 + TSAN_OPTIONS: halt_on_error=1 + run: | + Result.Linux.x64.Debug/ZEngine/tests/ZEngineTests \ + --gtest_filter='CookedObjectSnapshotTest.*:CookedPackageSnapshotTest.*:SceneLoadCoordinatorTest.*:SceneRuntimeCameraTest.*:SceneSubmissionProgressTest.*:SceneInstanceTest.*:SceneResourceBundleTest.*:SceneEnvironmentHandoffTest.*' \ + --gtest_output=xml:scene-sanitizer-results.xml + python3 tests/scene/verify_scene_sanitizer_results.py scene-sanitizer-results.xml diff --git a/.github/workflows/job-test-windows.yml b/.github/workflows/job-test-windows.yml index afffbc2aa..699879136 100644 --- a/.github/workflows/job-test-windows.yml +++ b/.github/workflows/job-test-windows.yml @@ -53,4 +53,10 @@ jobs: shell: pwsh env: PATH: ${{github.workspace}}\lib - run: ${{github.workspace}}\tests\TetragramaTests.exe + run: | + & "${{github.workspace}}\tests\TetragramaTests.exe" + if ($LASTEXITCODE -ne 0) { + $nativeCode = [uint32]([int64]$LASTEXITCODE -band 0xFFFFFFFFL) + Write-Error ("Editor test process failed with native exit code 0x{0:X8}" -f $nativeCode) + exit 1 + } diff --git a/Obelisk/CMakeLists.txt b/Obelisk/CMakeLists.txt index 5743c0e58..f7f0bcbcb 100644 --- a/Obelisk/CMakeLists.txt +++ b/Obelisk/CMakeLists.txt @@ -63,3 +63,55 @@ endif() install(TARGETS ${TARGET_NAME} DESTINATION bin) zengine_sign_installed_target(${TARGET_NAME} bin) + +# These tests create real windows and submit Vulkan work. Keep them opt-in so +# headless builders cannot silently substitute a non-rendering test for them. +option(ZENGINE_ENABLE_COOKED_DESKTOP_TESTS "Register cooked scene desktop Vulkan process tests" OFF) +if(BUILD_TESTING AND ZENGINE_ENABLE_COOKED_DESKTOP_TESTS) + find_package(Python3 REQUIRED COMPONENTS Interpreter) + set(_cooked_smoke "${CMAKE_SOURCE_DIR}/tests/scene/cooked_scene_smoke.py") + set(_cooked_common "${Python3_EXECUTABLE}" "${_cooked_smoke}" + --cooker "$" --obelisk "$") + add_test(NAME CookedDesktop.StartupAndPackageFailures COMMAND ${_cooked_common} --failure-cases) + add_test(NAME CookedDesktop.Replacements COMMAND ${_cooked_common} --reloads 3) + add_test(NAME CookedDesktop.RuntimeMainCamera COMMAND ${_cooked_common} --runtime-camera main) + add_test(NAME CookedDesktop.RuntimeExplicitCamera COMMAND ${_cooked_common} --runtime-camera a0000000-0000-0000-0000-00000000000a) + add_test(NAME CookedDesktop.StandaloneMainCamera COMMAND ${_cooked_common} --standalone --runtime-camera main) + add_test(NAME CookedDesktop.StandalonePackageFailures COMMAND ${_cooked_common} --standalone --runtime-camera main --failure-cases --audit-source --require-validation) + add_test(NAME CookedDesktop.StandaloneCapture COMMAND ${_cooked_common} --standalone --runtime-camera main --verify-capture) + add_test(NAME CookedDesktop.StandaloneSourceIndependence COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 3 --audit-source) + add_test(NAME CookedDesktop.StandaloneExplicitCamera COMMAND ${_cooked_common} --standalone --runtime-camera a0000000-0000-0000-0000-00000000000a) + add_test(NAME CookedDesktop.StandaloneMissingCamera COMMAND ${_cooked_common} --standalone --runtime-camera b0000000-0000-0000-0000-00000000000b --reject-runtime-camera) + add_test(NAME CookedDesktop.StandaloneReplacements COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 3) + add_test(NAME CookedDesktop.StandaloneLifecycleStressSmoke COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 6 --lifecycle-stress) + add_test(NAME CookedDesktop.StandaloneSteadyAndReplacement COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --steady-frames 30) + add_test(NAME CookedDesktop.StandaloneMissingReplacementRetry COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --missing-replacement --retry-after-failure) + add_test(NAME CookedDesktop.StandaloneMissingReplacementCameraRetry COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --missing-replacement-camera --retry-after-failure) + add_test(NAME CookedDesktop.StandaloneCorruptReplacementMaterialRetry COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --corrupt-replacement-material --retry-after-failure --require-validation --audit-source) + add_test(NAME CookedDesktop.StandaloneMissingReplacementMaterialRetry COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --missing-replacement-material --retry-after-failure --require-validation --audit-source) + add_test(NAME CookedDesktop.StandaloneUploadCancellationRetry COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --cancel-during-upload --retry-after-failure --require-validation --audit-source) + add_test(NAME CookedDesktop.StandaloneCpuCancellationRetry COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --cancel-during-cpu --retry-after-failure --require-validation --audit-source) + add_test(NAME CookedDesktop.StandaloneUploadSubmissionRejectionRetry COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --reject-submit --retry-after-failure --require-validation --audit-source) + add_test(NAME CookedDesktop.StandaloneStagingRefusalRetry COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --fail-staging --retry-after-failure) + add_test(NAME CookedDesktop.StandalonePreflightRefusalRetry COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --fail-preflight --retry-after-failure --require-validation --audit-source) + add_test(NAME CookedDesktop.StandaloneDeviceLoss COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --device-loss) + add_test(NAME CookedDesktop.StandaloneShutdownCpu COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --shutdown-cpu --require-validation) + add_test(NAME CookedDesktop.StandaloneShutdownGpu COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --shutdown-gpu --require-validation) + add_test(NAME CookedDesktop.StandaloneCorruptReplacementRetry COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --corrupt-replacement --retry-after-failure) + add_test(NAME CookedDesktop.StandaloneSupersedeCpu COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --supersede-during-cpu) + add_test(NAME CookedDesktop.StandaloneSupersedeWaiting COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --supersede-reload) + add_test(NAME CookedDesktop.StandaloneSupersedeUpload COMMAND ${_cooked_common} --standalone --runtime-camera main --reloads 1 --supersede-during-upload) + add_test(NAME CookedDesktop.RuntimeMissingCamera COMMAND ${_cooked_common} --runtime-camera b0000000-0000-0000-0000-00000000000b --reject-runtime-camera) + add_test(NAME CookedDesktop.SupersedeWaiting COMMAND ${_cooked_common} --reloads 1 --supersede-reload) + add_test(NAME CookedDesktop.SupersedeCpu COMMAND ${_cooked_common} --reloads 1 --supersede-during-cpu) + add_test(NAME CookedDesktop.SupersedeUpload COMMAND ${_cooked_common} --reloads 1 --supersede-during-upload) + add_test(NAME CookedDesktop.CpuCancellationRetry COMMAND ${_cooked_common} --reloads 1 --cancel-during-cpu --retry-after-failure) + add_test(NAME CookedDesktop.UploadCancellationRetry COMMAND ${_cooked_common} --reloads 1 --cancel-during-upload --retry-after-failure) + add_test(NAME CookedDesktop.StagingRefusalRetry COMMAND ${_cooked_common} --reloads 1 --fail-staging --retry-after-failure) + add_test(NAME CookedDesktop.UploadSubmissionRejectionRetry COMMAND ${_cooked_common} --reloads 1 --reject-submit --retry-after-failure) + add_test(NAME CookedDesktop.CorruptReplacementRetry COMMAND ${_cooked_common} --reloads 1 --corrupt-replacement --retry-after-failure) + set_tests_properties(CookedDesktop.StartupAndPackageFailures CookedDesktop.Replacements CookedDesktop.RuntimeMainCamera CookedDesktop.RuntimeExplicitCamera CookedDesktop.StandaloneMainCamera CookedDesktop.StandaloneCapture CookedDesktop.StandaloneSourceIndependence CookedDesktop.StandaloneExplicitCamera CookedDesktop.StandaloneMissingCamera CookedDesktop.StandaloneReplacements CookedDesktop.StandaloneSteadyAndReplacement CookedDesktop.StandaloneMissingReplacementRetry CookedDesktop.StandaloneUploadCancellationRetry CookedDesktop.StandaloneCpuCancellationRetry CookedDesktop.StandaloneUploadSubmissionRejectionRetry CookedDesktop.StandaloneStagingRefusalRetry CookedDesktop.StandaloneDeviceLoss CookedDesktop.StandaloneCorruptReplacementRetry CookedDesktop.StandaloneSupersedeWaiting CookedDesktop.StandaloneSupersedeCpu CookedDesktop.StandaloneSupersedeUpload CookedDesktop.RuntimeMissingCamera CookedDesktop.SupersedeWaiting CookedDesktop.SupersedeCpu CookedDesktop.SupersedeUpload + CookedDesktop.StandalonePackageFailures CookedDesktop.StandaloneLifecycleStressSmoke CookedDesktop.StandaloneShutdownCpu CookedDesktop.StandaloneShutdownGpu CookedDesktop.StandalonePreflightRefusalRetry CookedDesktop.StandaloneMissingReplacementCameraRetry CookedDesktop.StandaloneCorruptReplacementMaterialRetry CookedDesktop.StandaloneMissingReplacementMaterialRetry CookedDesktop.CpuCancellationRetry CookedDesktop.UploadCancellationRetry + CookedDesktop.StagingRefusalRetry CookedDesktop.UploadSubmissionRejectionRetry CookedDesktop.CorruptReplacementRetry + PROPERTIES TIMEOUT 240 RUN_SERIAL TRUE LABELS "cook;scene;desktop;vulkan") +endif() diff --git a/Obelisk/EntryPoint.cpp b/Obelisk/EntryPoint.cpp index 418d6324d..cc86d5f3a 100644 --- a/Obelisk/EntryPoint.cpp +++ b/Obelisk/EntryPoint.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -8,6 +9,8 @@ #include #include #include +#include +#include #include #ifdef ZENGINE_PLATFORM @@ -28,18 +31,196 @@ int applicationEntryPoint(int argc, char* argv[]) CrashHandler::Install("Obelisk", "1.0.0", crash_log_directory_string.c_str()); CLI::App cli{"ObeliskCLI"}; - argv = cli.ensure_utf8(argv); - std::string config_file = ""; - bool launch_editor = false; - bool gizmo_debug = false; + argv = cli.ensure_utf8(argv); + std::string config_file = ""; + std::string cooked_package = ""; + std::string cooked_scene = ""; + std::string cooked_capture = ""; + std::string source_capture = ""; + std::string source_import = ""; + std::string log_directory_override = ""; + std::string cooked_alternate = ""; + std::string cooked_stage_trace = ""; + std::string cooked_camera = ""; + bool launch_editor = false; + bool gizmo_debug = false; + bool cooked_exit_after_frame = false; + uint32_t cooked_expected_draws = 0; + uint32_t cooked_reloads = 0; + uint32_t cooked_steady_frames = 0; + uint32_t cooked_post_steady_frames = 0; + bool cooked_cancel_reload = false; + bool cooked_cancel_cpu = false; + bool cooked_retry_failure = false; + bool cooked_supersede_reload = false; + bool cooked_supersede_cpu = false; + bool cooked_supersede_gpu = false; + bool cooked_runtime_view = false; + bool cooked_standalone = false; + bool cooked_fail_reload_staging = false; + bool cooked_fail_preflight = false; + bool cooked_reject_reload_submit = false; + bool cooked_device_loss = false; + bool cooked_audit_source = false; + bool cooked_lifecycle_stress = false; + bool cooked_fixed_window = false; + bool cooked_wait_visible = false; + bool cooked_observe_gpu_completion = false; + bool cooked_shutdown_cpu = false; + bool cooked_shutdown_gpu = false; cli.add_option("--projectConfigFile", config_file, "The project config file"); + cli.add_option("--cookedPackage", cooked_package, "Read-only cooked package directory"); + cli.add_option("--cookedScene", cooked_scene, "Scene UUID in the cooked package"); + cli.add_flag("--cookedRuntimeView", cooked_runtime_view, "Use the cooked scene's authored camera in the editor-hosted full-window view"); + cli.add_flag("--cookedStandalone", cooked_standalone, "Run a cooked scene in a renderer-only runtime host"); + cli.add_option("--cookedCamera", cooked_camera, "Optional authored camera UUID; otherwise require exactly one main camera"); + cli.add_flag("--cookedExitAfterFrame", cooked_exit_after_frame, "Exit after the cooked scene's first recorded frame (test runs)"); + cli.add_option("--cookedExpectedDraws", cooked_expected_draws, "Require this many prepared and submitted draws in a cooked test run"); + cli.add_option("--cookedReloads", cooked_reloads, "Number of alternating cooked scene replacements in a test run"); + cli.add_option("--cookedSteadyFrames", cooked_steady_frames, "Measure this many settled standalone frames before test replacements"); + cli.add_option("--cookedPostSteadyFrames", cooked_post_steady_frames, "Measure settled standalone frames after test replacements"); + cli.add_option("--cookedAlternateScene", cooked_alternate, "Second scene UUID for cooked replacement test runs"); + cli.add_flag("--cookedCancelReload", cooked_cancel_reload, "Cancel the first replacement after recording its mesh upload"); + cli.add_flag("--cookedCancelCpu", cooked_cancel_cpu, "Cancel the first replacement while its CPU worker owns the reconstruction"); + cli.add_flag("--cookedRetryFailure", cooked_retry_failure, "Retry the original cooked scene after a replacement failure"); + cli.add_flag("--cookedSupersedeReload", cooked_supersede_reload, "Supersede the first waiting cooked replacement"); + cli.add_flag("--cookedSupersedeCpu", cooked_supersede_cpu, "Supersede a replacement while its CPU worker owns resources"); + cli.add_flag("--cookedSupersedeGpu", cooked_supersede_gpu, "Supersede a replacement after mesh upload recording"); + cli.add_flag("--cookedFailReloadStaging", cooked_fail_reload_staging, "Refuse new upload staging allocations during the first replacement"); + cli.add_flag("--cookedFailPreflight", cooked_fail_preflight, "Refuse a ready replacement before publication (test runs)"); + cli.add_flag("--cookedRejectReloadSubmit", cooked_reject_reload_submit, "Reject the first replacement's recorded cooked upload batch (test runs)"); + cli.add_flag("--cookedInjectDeviceLoss", cooked_device_loss, "Inject device loss during a standalone replacement upload (test runs)"); + cli.add_flag("--cookedAuditSource", cooked_audit_source, "Deny and count source VFS, registry and import lookups during standalone loading (test runs)"); + cli.add_flag("--cookedLifecycleStress", cooked_lifecycle_stress, "Alternate CPU and GPU upload cancellations with scene replacement (test runs)"); + cli.add_flag("--cookedFixedWindow", cooked_fixed_window, "Use a fixed 1280x720 logical window (test runs)"); + cli.add_flag("--cookedWaitVisible", cooked_wait_visible, "Wait for the fixed test window to become visible before loading"); + cli.add_flag("--cookedObserveGpuCompletion", cooked_observe_gpu_completion, "Observe completion of each generation's first accepted render submission (test runs)"); + cli.add_flag("--cookedShutdownCpu", cooked_shutdown_cpu, "Close while a replacement CPU worker is active (test runs)"); + cli.add_flag("--cookedShutdownGpu", cooked_shutdown_gpu, "Close after recording a replacement GPU upload (test runs)"); + cli.add_option("--cookedCapture", cooked_capture, "Write a bounded RGB8 PPM scene viewport capture in a cooked test run"); + cli.add_option("--cookedStageTrace", cooked_stage_trace, "Write a bounded SceneLoad stage CSV for a cooked test run"); + cli.add_option("--sourceCapture", source_capture, "Capture the configured source scene for a test run"); + cli.add_option("--sourceImport", source_import, "Import this source asset before the source capture scene"); + cli.add_option("--logDirectory", log_directory_override, "Write this process's engine logs to a specified directory"); cli.add_option("--launchEditor", launch_editor, "Activate the editor"); cli.add_flag("--gizmoDebug", gizmo_debug, "Show the non-interactive native gizmo renderer fixture (editor only)"); CLI11_PARSE(cli, argc, argv); + if (cooked_fixed_window && !((cooked_standalone && cooked_exit_after_frame) || (launch_editor && !source_capture.empty() && cooked_runtime_view))) + { + std::fprintf(stderr, "Fixed test window requires a standalone run or an editor runtime-view source capture.\n"); + return 2; + } + + if (cooked_wait_visible && (!cooked_standalone || !cooked_exit_after_frame || !cooked_fixed_window)) + { + std::fprintf(stderr, "Visible-window gating requires a fixed standalone test.\n"); + return 2; + } +#if !defined(__APPLE__) || !ZENGINE_PROFILING + if (cooked_wait_visible) + { + std::fprintf(stderr, "Visible-window gating requires a profiled macOS build.\n"); + return 2; + } +#endif + + if (cooked_observe_gpu_completion && (!cooked_standalone || !cooked_exit_after_frame || cooked_device_loss || cooked_shutdown_cpu || cooked_shutdown_gpu)) + { + std::fprintf(stderr, "Render completion observation requires a standalone non-fatal test run.\n"); + return 2; + } + + if (cooked_post_steady_frames && (!cooked_standalone || !cooked_exit_after_frame || !cooked_reloads || cooked_post_steady_frames > 600 || cooked_lifecycle_stress || cooked_retry_failure || cooked_device_loss || cooked_shutdown_cpu || cooked_shutdown_gpu)) + { + std::fprintf(stderr, "Post-replacement steady observation requires a bounded standalone replacement run.\n"); + return 2; + } + + if (cooked_fail_preflight && (!cooked_standalone || cooked_reloads != 1 || !cooked_retry_failure || cooked_cancel_reload || cooked_cancel_cpu || cooked_fail_reload_staging || cooked_supersede_reload || cooked_supersede_cpu || cooked_supersede_gpu || cooked_reject_reload_submit || cooked_device_loss || cooked_lifecycle_stress || cooked_shutdown_cpu || cooked_shutdown_gpu)) + { + std::fprintf(stderr, "Publication preflight refusal requires one standalone replacement and retry without another probe.\n"); + return 2; + } + + if ((cooked_shutdown_cpu || cooked_shutdown_gpu) && (!cooked_standalone || !cooked_exit_after_frame || cooked_reloads != 1 || (cooked_shutdown_cpu && cooked_shutdown_gpu) || cooked_lifecycle_stress || cooked_cancel_reload || cooked_cancel_cpu || cooked_retry_failure || cooked_supersede_reload || cooked_supersede_cpu || cooked_supersede_gpu || cooked_fail_reload_staging || cooked_reject_reload_submit || cooked_device_loss || cooked_steady_frames || !cooked_capture.empty())) + { + std::fprintf(stderr, "Shutdown probe requires one standalone replacement without other probes.\n"); + return 2; + } + + if (cooked_lifecycle_stress && (!cooked_standalone || !cooked_exit_after_frame || !cooked_reloads || cooked_cancel_reload || cooked_cancel_cpu || cooked_retry_failure || cooked_supersede_reload || cooked_supersede_cpu || cooked_supersede_gpu || cooked_fail_reload_staging || cooked_reject_reload_submit || cooked_device_loss || cooked_steady_frames || !cooked_capture.empty())) + { + std::fprintf(stderr, "Lifecycle stress requires standalone replacements without other probes.\n"); + return 2; + } + + if (cooked_audit_source && (!cooked_standalone || !cooked_exit_after_frame)) + { + std::fprintf(stderr, "Source-independence audit requires a standalone test run.\n"); + return 2; + } + + if (cooked_device_loss && (!cooked_standalone || cooked_reloads != 1 || !cooked_exit_after_frame || cooked_cancel_reload || cooked_cancel_cpu || cooked_fail_reload_staging || cooked_retry_failure || cooked_supersede_reload || cooked_supersede_cpu || cooked_supersede_gpu || cooked_reject_reload_submit || cooked_steady_frames || !cooked_capture.empty())) + { + std::fprintf(stderr, "Device-loss probe requires one standalone replacement without other probes.\n"); + return 2; + } + + if (cooked_steady_frames && (!cooked_standalone || !cooked_exit_after_frame || cooked_steady_frames > 600 || cooked_cancel_reload || cooked_cancel_cpu || cooked_fail_reload_staging || cooked_retry_failure || cooked_supersede_reload || cooked_supersede_cpu || cooked_supersede_gpu || cooked_reject_reload_submit || !cooked_capture.empty())) + { + std::fprintf(stderr, "Steady frame measurement requires a standalone test without fault probes or capture and is limited to 600 frames.\n"); + return 2; + } + + if (cooked_standalone && (cooked_cancel_reload || cooked_cancel_cpu || cooked_fail_reload_staging) && !cooked_retry_failure) + { + std::fprintf(stderr, "Standalone cancellation probe requires --cookedRetryFailure.\n"); + return 2; + } + + if (cooked_reject_reload_submit && (cooked_reloads != 1 || !cooked_retry_failure || cooked_cancel_reload || cooked_cancel_cpu || cooked_fail_reload_staging || cooked_supersede_reload || cooked_supersede_cpu || cooked_supersede_gpu)) + { + std::fprintf(stderr, "Cooked submission rejection requires one replacement and retry, without another fault probe.\n"); + return 2; + } + + const auto cooked_uuid = uuids::uuid::from_string(cooked_scene); + const auto alternate_uuid = uuids::uuid::from_string(cooked_alternate); + const auto camera_uuid = uuids::uuid::from_string(cooked_camera); + if ((launch_editor == cooked_standalone) || (cooked_standalone && (cooked_package.empty() || cooked_runtime_view || (cooked_retry_failure && cooked_reloads != 1) || gizmo_debug || !source_capture.empty() || !source_import.empty())) || (cooked_package.empty() != cooked_scene.empty()) || (cooked_exit_after_frame && cooked_package.empty()) || (!source_import.empty() && source_capture.empty()) || ((cooked_cancel_reload || cooked_fail_reload_staging || cooked_cancel_cpu) && cooked_reloads != 1) || + (int(cooked_cancel_reload) + int(cooked_fail_reload_staging) + int(cooked_cancel_cpu) > 1) || (cooked_reloads && (!cooked_exit_after_frame || !alternate_uuid || alternate_uuid->is_nil() || !cooked_uuid || *alternate_uuid == *cooked_uuid)) || (!cooked_reloads && !cooked_alternate.empty()) || cooked_reloads > (cooked_lifecycle_stress ? 1100u : 1000u) || ((cooked_expected_draws || !cooked_capture.empty()) && !cooked_exit_after_frame && source_capture.empty()) || + (!source_capture.empty() && (config_file.empty() || !cooked_package.empty() || cooked_exit_after_frame || !cooked_capture.empty())) || (!cooked_package.empty() && (!config_file.empty() || !cooked_uuid || cooked_uuid->is_nil()))) + { + std::fprintf(stderr, "Choose --launchEditor true or --cookedStandalone; cooked launch requires --cookedPackage and --cookedScene UUID, without --projectConfigFile.\n"); + return 2; + } + if (!cooked_stage_trace.empty() && (!cooked_exit_after_frame || std::filesystem::exists(cooked_stage_trace))) + { + std::fprintf(stderr, "Cooked stage trace requires a cooked test run and a new output path.\n"); + return 2; + } + if ((cooked_supersede_reload || cooked_supersede_cpu || cooked_supersede_gpu) && (cooked_reloads != 1 || int(cooked_supersede_reload) + int(cooked_supersede_cpu) + int(cooked_supersede_gpu) > 1 || cooked_cancel_reload || cooked_cancel_cpu || cooked_fail_reload_staging || cooked_retry_failure)) + { + std::fprintf(stderr, "Cooked supersession probe requires one replacement without another fault probe.\n"); + return 2; + } + if ((cooked_runtime_view && cooked_package.empty() && source_capture.empty()) || (!cooked_camera.empty() && (!(cooked_runtime_view || cooked_standalone) || !camera_uuid || camera_uuid->is_nil()))) + { + std::fprintf(stderr, "Runtime view requires a cooked package or source capture; an explicit camera must be a non-nil UUID.\n"); + return 2; + } +#if !ZENGINE_PROFILING || ZENGINE_TRACY + if (!cooked_stage_trace.empty()) + { + std::fprintf(stderr, "Cooked stage CSV requires non-Tracy profiling in this build.\n"); + return 2; + } +#endif + MemoryManager manager = {}; - manager.Initialize(ZGiga(8ULL), launch_editor ? MemoryBudgetConfig::Editor() : MemoryBudgetConfig::Default()); + manager.Initialize(ZGiga(8ULL), MemoryBudgetConfig::Editor()); #if ZENGINE_PROFILING ZEngine::Profiling::MemoryProfiler::Initialize(&manager.BootstrapArena); ZEngine::Profiling::MemoryProfiler::TrackArena("Bootstrap", &manager.BootstrapArena); @@ -49,8 +230,10 @@ int applicationEntryPoint(int argc, char* argv[]) ArenaAllocator logger_arena = {}; LoggerConfiguration logger_cfg = {}; - logger_cfg.OutputDirectory = log_directory.string(); + logger_cfg.OutputDirectory = log_directory_override.empty() ? log_directory.string() : log_directory_override; logger_cfg.CrashLogDir = crash_log_directory_string; + if (cooked_lifecycle_stress) + logger_cfg.FileMaxBytes = 64 * 1024 * 1024; manager.CreateBudgetedArena(manager.Budget.Logging, &logger_arena); Logger::Initialize(&logger_arena, logger_cfg); CrashHandler::SetPreCrashCallback([](void*) { @@ -69,14 +252,91 @@ int applicationEntryPoint(int argc, char* argv[]) { app = ZPushStructCtor(arena); static_cast(app)->EnableGizmoDebug = gizmo_debug; - app->EnableRenderOverlay = true; + if (!cooked_package.empty()) + static_cast(app)->ConfigureCookedStartup(cooked_package, *cooked_uuid, cooked_exit_after_frame, cooked_expected_draws, cooked_capture, cooked_reloads, alternate_uuid.value_or(uuids::uuid{}), cooked_cancel_reload, cooked_fail_reload_staging, cooked_cancel_cpu); + if (cooked_runtime_view) + static_cast(app)->ConfigureCookedRuntimeView(camera_uuid.value_or(uuids::uuid{})); + if (cooked_retry_failure) + static_cast(app)->EnableCookedRetryProbe(); + if (cooked_reject_reload_submit) + static_cast(app)->EnableCookedSubmitRejectionProbe(); + if (cooked_supersede_reload) + static_cast(app)->EnableCookedSupersedeProbe(); + if (cooked_supersede_cpu) + static_cast(app)->EnableCookedCpuSupersedeProbe(); + if (cooked_supersede_gpu) + static_cast(app)->EnableCookedGpuSupersedeProbe(); + if (!source_capture.empty()) + static_cast(app)->ConfigureSourceCapture(source_capture, cooked_expected_draws, source_import); + app->EnableRenderOverlay = true; + } + else + { + auto* runtime = ZPushStructCtor(arena); + runtime->Configure(cooked_package, *cooked_uuid, camera_uuid.value_or(uuids::uuid{}), cooked_exit_after_frame, cooked_expected_draws, cooked_reloads, alternate_uuid.value_or(uuids::uuid{}), cooked_retry_failure, cooked_cancel_reload, cooked_cancel_cpu, cooked_supersede_cpu, cooked_supersede_gpu, cooked_supersede_reload); + if (cooked_lifecycle_stress) + runtime->EnableLifecycleStress(); + if (cooked_fixed_window) + runtime->EnableFixedTestWindow(); + if (cooked_observe_gpu_completion) + runtime->EnableGpuCompletionObservation(); + if (cooked_shutdown_cpu || cooked_shutdown_gpu) + runtime->EnableShutdownProbe(cooked_shutdown_gpu); + if (cooked_fail_preflight) + runtime->EnablePreflightRefusalProbe(); + if (cooked_steady_frames) + runtime->ConfigureSteadyFrames(cooked_steady_frames); + if (cooked_post_steady_frames) + runtime->ConfigurePostSteadyFrames(cooked_post_steady_frames); + if (cooked_reject_reload_submit) + runtime->EnableUploadSubmissionRejectionProbe(); + if (cooked_fail_reload_staging) + runtime->EnableStagingRefusalProbe(); + if (cooked_device_loss) + runtime->EnableDeviceLossProbe(); + if (cooked_audit_source) + runtime->EnableSourceIndependenceAudit(); + if (!cooked_capture.empty()) + runtime->ConfigureCapture(cooked_capture); + app = runtime; } app->ConfigFile = config_file_str; + if (cooked_fixed_window) + { + app->WindowCfg.Width = 1280; + app->WindowCfg.Height = 720; + app->WindowCfg.Resizable = false; + app->WindowCfg.Maximize = false; + } + app->WindowCfg.WaitForVisibleTestWindow = cooked_wait_visible; + bool trace_ok = true; +#if ZENGINE_PROFILING && !ZENGINE_TRACY + if (!cooked_stage_trace.empty()) + trace_ok = ZEngine::Profiling::ProfilerBuffer::StartTimedCapture(120000, "SceneLoad."); +#endif app->Initialize(&manager); app->Run(); +#if ZENGINE_PROFILING && !ZENGINE_TRACY + if (!cooked_stage_trace.empty()) + { + const bool expired = ZEngine::Profiling::ProfilerBuffer::CaptureDeadlinePassed(); + ZEngine::Profiling::ProfilerBuffer::StopCapture(); + const auto dropped = ZEngine::Profiling::ProfilerBuffer::CaptureDropped(); + const bool exported = ZEngine::Profiling::ProfilerBuffer::WriteCaptureCsv(cooked_stage_trace.c_str()); + std::fprintf(stderr, "Cooked stage capture: records=%u dropped=%u expired=%u exported=%u\n", ZEngine::Profiling::ProfilerBuffer::CaptureCount(), dropped, static_cast(expired), static_cast(exported)); + trace_ok = trace_ok && !dropped && !expired && exported; + } +#endif + const bool cooked_frame_recorded = !(cooked_exit_after_frame || !source_capture.empty()) || (cooked_standalone ? static_cast(app)->FrameRecorded() : static_cast(app)->CookedStartupFrameRecorded()); + const bool cooked_failure_preserved = !cooked_standalone && cooked_exit_after_frame && static_cast(app)->CookedStartupFailurePreserved(); + const bool cancellation_observed = !cooked_cancel_reload || (cooked_standalone ? static_cast(app)->UploadCancellationObserved() : static_cast(app)->CookedReloadCancelledDuringUpload()); + const bool cpu_cancellation_observed = !cooked_cancel_cpu || (cooked_standalone ? static_cast(app)->CpuCancellationObserved() : static_cast(app)->CookedReloadCancelledDuringCpu()); + const bool staging_refusal_observed = !cooked_fail_reload_staging || (cooked_standalone ? static_cast(app)->StagingRefusalObserved() : static_cast(app)->CookedReloadStagingRefused()); app->Shutdown(); + const bool source_audit_passed = !cooked_audit_source || static_cast(app)->SourceAuditPassed(); + const bool shutdown_probe_passed = !(cooked_shutdown_cpu || cooked_shutdown_gpu) || static_cast(app)->ShutdownProbePassed(); // Step 16 — flush and dispose logger Logger::Flush(); @@ -89,7 +349,7 @@ int applicationEntryPoint(int argc, char* argv[]) manager.Shutdown(); CrashHandler::Uninstall(); - return 0; + return !trace_ok || !source_audit_passed || !shutdown_probe_passed ? 2 : cooked_frame_recorded ? 0 : cooked_failure_preserved && cancellation_observed && cpu_cancellation_observed && staging_refusal_observed ? 1 : 2; } #ifdef _WIN32 diff --git a/Scripts/ui-fidelity/capture.sh b/Scripts/ui-fidelity/capture.sh new file mode 100755 index 000000000..bddf25b1a --- /dev/null +++ b/Scripts/ui-fidelity/capture.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Launch the editor, capture its window, and quit it. +# capture.sh [settle-seconds] +# Output: (window only, no shadow, native resolution) and .json with the +# window's logical size. Requires macOS Screen Recording permission for the calling terminal. +set -euo pipefail +BIN="$1"; PROJECT="$2"; OUT="$3"; SETTLE="${4:-14}" +HERE="$(cd "$(dirname "$0")" && pwd)" + +pkill -x Obelisk 2>/dev/null || true +# A saved dock layout overrides the code defaults; start from the defaults. +rm -f "$(dirname "$BIN")/ZodiacEngine/Settings/zui_layout.ini" +( cd "$(dirname "$BIN")" && "$BIN" --launchEditor true --projectConfigFile "$PROJECT" >/tmp/obelisk_capture.log 2>&1 & ) +sleep "$SETTLE" + +# Size the window so its content area is exactly the 1600x1000 Figma frame (32 px title bar on top). +# Needs a display with room for it: WIN_POS is the top-left position in AppleScript coordinates +# (default: the external 2560x1440 display to the right of the laptop screen). +WIN_POS="${WIN_POS:-1600,450}"; WIN_SIZE="${WIN_SIZE:-1600,1032}" +osascript -e "tell application \"System Events\" to tell process \"Obelisk\" to set position of window 1 to {${WIN_POS}}" \ + -e "tell application \"System Events\" to tell process \"Obelisk\" to set size of window 1 to {${WIN_SIZE}}" >/dev/null 2>&1 || echo "could not resize window (Accessibility permission?)" >&2 +sleep 3 + +INFO="$(osascript -l JavaScript "$HERE/window-bounds.js")" +# Optional clicks ("x,y;x,y" in window-content logical px) to put the editor in a state, e.g. select an actor. +osascript -e 'tell application "System Events" to set frontmost of process "Obelisk" to true' >/dev/null 2>&1; sleep 1 +if [ -n "${CLICKS:-}" ]; then + WX="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])[0]["x"])' "$INFO")" + WY="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])[0]["y"])' "$INFO")" + IFS=';' read -ra PTS <<< "$CLICKS" + for pt in "${PTS[@]}"; do + BTN=""; case "$pt" in r*) BTN="r"; pt="${pt#r}";; esac + CX="${pt%,*}"; CY="${pt#*,}" + osascript -l JavaScript "$HERE/click.js" "$(python3 -c "print($WX + $CX)")" "$(python3 -c "print($WY + 32 + $CY)")" $BTN >/dev/null 2>&1 + sleep 1 + done + sleep 1 +fi +ID="$(python3 -c 'import json,sys; w=json.loads(sys.argv[1]); print(w[0]["id"] if w else "")' "$INFO")" +if [ -z "$ID" ]; then echo "no Obelisk window found" >&2; pkill -x Obelisk || true; exit 1; fi +screencapture -o -x -l "$ID" "$OUT" +echo "$INFO" > "$OUT.json" +pkill -x Obelisk || true +echo "captured window $ID -> $OUT" diff --git a/Scripts/ui-fidelity/click.js b/Scripts/ui-fidelity/click.js new file mode 100644 index 000000000..f1643cbbf --- /dev/null +++ b/Scripts/ui-fidelity/click.js @@ -0,0 +1,15 @@ +// osascript -l JavaScript click.js : left click at a screen position. +ObjC.import('CoreGraphics'); +function run(argv) { + const x = parseFloat(argv[0]), y = parseFloat(argv[1]); + const p = {x: x, y: y}; + const mv = $.CGEventCreateMouseEvent(null, $.kCGEventMouseMoved, p, 0); + $.CGEventPost($.kCGHIDEventTap, mv); + delay(0.3); + const right = argv[2] === 'r'; + const dn = $.CGEventCreateMouseEvent(null, right ? $.kCGEventRightMouseDown : $.kCGEventLeftMouseDown, p, right ? 1 : 0); + $.CGEventPost($.kCGHIDEventTap, dn); + delay(0.08); + const up = $.CGEventCreateMouseEvent(null, right ? $.kCGEventRightMouseUp : $.kCGEventLeftMouseUp, p, right ? 1 : 0); + $.CGEventPost($.kCGHIDEventTap, up); +} diff --git a/Scripts/ui-fidelity/compare.py b/Scripts/ui-fidelity/compare.py new file mode 100644 index 000000000..02f0ba9e5 --- /dev/null +++ b/Scripts/ui-fidelity/compare.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Compare a captured editor window against the Figma reference frame. + + compare.py [--reference reference/editor-3_2503.png] + +The window is cropped below the macOS title bar, scaled to logical pixels, and compared with the +1600x1000 Figma frame in regions anchored to the nearest window corner (the editor's chrome is +fixed-size, so anchored regions line up regardless of window size). Per region it prints the mean +absolute colour error (0-255, lower is better) and the share of pixels off by more than 24. +Side-by-side images (reference | capture | difference) are written to . +""" +import argparse, json, os, sys +import numpy as np +from PIL import Image + +# name, anchor, (x0, y0, x1, y1) in Figma frame coordinates (1600x1000) +REGIONS = [ + ("menu-bar", "tl", (0, 0, 620, 28)), + ("level-tabs", "tl", (0, 28, 620, 58)), + ("main-toolbar", "tl", (0, 58, 900, 100)), + ("outliner", "tr", (1217, 100, 1600, 400)), + ("details-header", "tr", (1217, 400, 1600, 640)), + ("content-browser", "bl", (0, 684, 900, 974)), + ("status-bar", "bl", (0, 974, 900, 1000)), +] +FIGMA_W, FIGMA_H = 1600, 1000 + +def content_top(img): + """First row (below the title bar) whose centre pixels are dark UI, not the white title bar.""" + a = np.asarray(img.convert("L"), dtype=np.int32) + col = a[:, a.shape[1] // 2 - 40: a.shape[1] // 2 + 40].mean(axis=1) + for y in range(len(col)): + if col[y] < 90: + return y + return 0 + +def crop_region(img, anchor, rect, w, h): + x0, y0, x1, y1 = rect + if anchor[1] == "r": x0, x1 = x0 - FIGMA_W + w, x1 - FIGMA_W + w + if anchor[0] == "b": y0, y1 = y0 - FIGMA_H + h, y1 - FIGMA_H + h + x0, y0 = max(0, x0), max(0, y0) + return img.crop((x0, y0, min(w, x1), min(h, y1))) + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("capture"); ap.add_argument("out") + ap.add_argument("--reference", default=os.path.join(os.path.dirname(__file__), "reference", "editor-3_2503.png")) + args = ap.parse_args() + os.makedirs(args.out, exist_ok=True) + + info = json.load(open(args.capture + ".json"))[0] + cap = Image.open(args.capture).convert("RGB") + scale = cap.width / info["w"] + top = content_top(cap) + cap = cap.crop((0, top, cap.width, cap.height)) + lw, lh = round(cap.width / scale), round(cap.height / scale) + cap = cap.resize((lw, lh), Image.LANCZOS) + ref = Image.open(args.reference).convert("RGB") + cap.save(os.path.join(args.out, "capture-logical.png")) + print(f"window content {lw}x{lh} logical (scale {scale:g}, title bar {round(top/scale)}px)") + + rows, total = [], 0.0 + for name, anchor, rect in REGIONS: + r = crop_region(ref, anchor, rect, FIGMA_W, FIGMA_H) + c = crop_region(cap, anchor, rect, lw, lh) + w, h = min(r.width, c.width), min(r.height, c.height) + r, c = r.crop((0, 0, w, h)), c.crop((0, 0, w, h)) + d = np.abs(np.asarray(r, dtype=np.int32) - np.asarray(c, dtype=np.int32)) + mae = float(d.mean()); bad = float((d.max(axis=2) > 24).mean() * 100) + total += mae; rows.append((name, mae, bad)) + pad = 6 + sheet = Image.new("RGB", (w * 3 + pad * 2, h), (255, 0, 255)) + sheet.paste(r, (0, 0)); sheet.paste(c, (w + pad, 0)) + sheet.paste(Image.fromarray(np.clip(d * 3, 0, 255).astype("uint8")), (2 * (w + pad), 0)) + sheet.save(os.path.join(args.out, f"region-{name}.png")) + print(f"{'region':<18}{'MAE':>8}{'off>24 %':>11}") + for n, m, b in rows: print(f"{n:<18}{m:>8.1f}{b:>11.1f}") + print(f"{'mean':<18}{total/len(rows):>8.1f}") + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Scripts/ui-fidelity/run.sh b/Scripts/ui-fidelity/run.sh new file mode 100755 index 000000000..3f5bcf09d --- /dev/null +++ b/Scripts/ui-fidelity/run.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# One-shot: capture the editor and compare it with the Figma frame. +# run.sh [python] +# Writes /tmp/fid/.png and /tmp/fid/out-/ (see compare.py / zoom.py). +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +BUILD="${1:?build dir}"; TAG="${2:?tag}"; PY="${3:-/tmp/fidenv/bin/python}" +mkdir -p /tmp/fid +"$ROOT/Scripts/ui-fidelity/capture.sh" "$ROOT/$BUILD/Obelisk/Debug/Obelisk" "$ROOT/SampleProject/project.json" "/tmp/fid/$TAG.png" 14 2>&1 | grep -v Terminated || true +"$PY" "$ROOT/Scripts/ui-fidelity/compare.py" "/tmp/fid/$TAG.png" "/tmp/fid/out-$TAG" diff --git a/Scripts/ui-fidelity/window-bounds.js b/Scripts/ui-fidelity/window-bounds.js new file mode 100644 index 000000000..49d69ccdf --- /dev/null +++ b/Scripts/ui-fidelity/window-bounds.js @@ -0,0 +1,5 @@ +ObjC.import('CoreGraphics'); +ObjC.import('Foundation'); +const list = ObjC.deepUnwrap(ObjC.castRefToObject($.CGWindowListCopyWindowInfo(1<<0|1<<4, 0))); +const out = list.filter(w => w.kCGWindowOwnerName === 'Obelisk' && w.kCGWindowLayer === 0).map(w => ({id:w.kCGWindowNumber, x:w.kCGWindowBounds.X, y:w.kCGWindowBounds.Y, w:w.kCGWindowBounds.Width, h:w.kCGWindowBounds.Height, name:w.kCGWindowName})); +JSON.stringify(out); diff --git a/Scripts/ui-fidelity/zoom.py b/Scripts/ui-fidelity/zoom.py new file mode 100644 index 000000000..e9c5c1e9f --- /dev/null +++ b/Scripts/ui-fidelity/zoom.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Stack the Figma reference over the capture for one region, scaled up. + + zoom.py x0 y0 x1 y1 [--anchor tl|tr|bl|br] [--factor 2] [--name zoom] + +Coordinates are Figma-frame coordinates (1600x1000). For anchors other than tl they are measured +from that corner of the frame, exactly as compare.py does, so the same numbers work for both. +Reads /capture-logical.png written by compare.py; writes /.png. +""" +import argparse, os +from PIL import Image + +FW, FH = 1600, 1000 +ap = argparse.ArgumentParser() +ap.add_argument("out"); ap.add_argument("rect", nargs=4, type=int) +ap.add_argument("--anchor", default="tl"); ap.add_argument("--factor", type=int, default=2) +ap.add_argument("--name", default="zoom") +ap.add_argument("--reference", default=os.path.join(os.path.dirname(__file__), "reference", "editor-3_2503.png")) +a = ap.parse_args() +cap = Image.open(os.path.join(a.out, "capture-logical.png")).convert("RGB") +ref = Image.open(a.reference).convert("RGB") +x0, y0, x1, y1 = a.rect +def crop(img, w, h): + cx0, cx1, cy0, cy1 = x0, x1, y0, y1 + if a.anchor[1] == "r": cx0, cx1 = x0 - FW + w, x1 - FW + w + if a.anchor[0] == "b": cy0, cy1 = y0 - FH + h, y1 - FH + h + return img.crop((max(0, cx0), max(0, cy0), min(w, cx1), min(h, cy1))) +r, c = crop(ref, FW, FH), crop(cap, cap.width, cap.height) +w, h = min(r.width, c.width), min(r.height, c.height) +r, c = r.crop((0, 0, w, h)), c.crop((0, 0, w, h)) +f = a.factor +sheet = Image.new("RGB", (w * f, h * f * 2 + 6), (255, 0, 255)) +sheet.paste(r.resize((w * f, h * f), Image.LANCZOS), (0, 0)) +sheet.paste(c.resize((w * f, h * f), Image.LANCZOS), (0, h * f + 6)) +path = os.path.join(a.out, a.name + ".png"); sheet.save(path); print(path) diff --git a/Tetragrama/Editor.cpp b/Tetragrama/Editor.cpp index bbf865987..dc65bce15 100644 --- a/Tetragrama/Editor.cpp +++ b/Tetragrama/Editor.cpp @@ -9,8 +9,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -362,6 +364,60 @@ namespace Tetragrama } } + void Editor::ConfigureCookedStartup(std::string directory, const uuids::uuid& scene, bool exit_after_first_frame, uint32_t expected_draws, std::string capture_path, uint32_t reloads, const uuids::uuid& alternate_scene, bool cancel_reload_after_mesh, bool fail_reload_staging, bool cancel_reload_cpu) + { + m_cooked_package_directory = std::move(directory); + m_cooked_startup_scene = scene; + m_cooked_current_scene = scene; + m_cooked_alternate_scene = alternate_scene; + m_cooked_reloads_remaining = reloads; + m_cooked_cancel_after_mesh = cancel_reload_after_mesh; + m_cooked_cancel_cpu = cancel_reload_cpu; + m_cooked_fail_reload_staging = fail_reload_staging; + m_cooked_startup_exit_after_frame = exit_after_first_frame; + m_cooked_expected_draws = expected_draws; + m_cooked_capture_path = std::move(capture_path); + } + + void Editor::ConfigureSourceCapture(std::string capture_path, uint32_t expected_draws, std::string import_path) + { + m_source_capture = true; + m_cooked_startup_exit_after_frame = true; + m_cooked_expected_draws = expected_draws; + m_cooked_capture_path = std::move(capture_path); + m_source_capture_import_path = std::move(import_path); + } + + bool Editor::BeginCookedCapture(void* context, uint32_t width, uint32_t height, uint64_t generation) + { + auto& editor = *static_cast(context); + if (generation != editor.m_cooked_capture_generation || width != editor.m_cooked_capture_width || height != editor.m_cooked_capture_height) + { + uint32_t requested = 1; + editor.m_cooked_capture_state.compare_exchange_strong(requested, 4, std::memory_order_acq_rel); + return false; + } + uint32_t requested = 1; + return editor.m_cooked_capture_state.compare_exchange_strong(requested, 2, std::memory_order_acq_rel); + } + + void Editor::CompleteCookedCapture(const void* data, size_t size, void* context) + { + auto& editor = *static_cast(context); + if (!data || size != editor.m_cooked_capture_bytes || !editor.m_cooked_capture_pixels) + { + editor.m_cooked_capture_state.store(4, std::memory_order_release); + return; + } + std::memcpy(editor.m_cooked_capture_pixels.get(), data, size); + editor.m_cooked_capture_state.store(3, std::memory_order_release); + } + + void Editor::CancelCookedCapture(void* context) + { + static_cast(context)->m_cooked_capture_state.store(4, std::memory_order_release); + } + void Editor::OverrideWindowConfiguration() { const char* project_name = Configuration->ProjectName.empty() ? "ZEngine Editor" : Configuration->ProjectName.c_str(); @@ -428,14 +484,18 @@ namespace Tetragrama // Keep an absent default file as an empty-editor startup, but load // a present one through the normal transactional scene path. std::string directory = {}; - if (Configuration && !Configuration->ActiveSceneName.empty() && TrySceneDialogDirectory(Configuration->WorkingSpacePath.c_str(), Configuration->ScenePath.c_str(), {}, &directory)) + if (m_cooked_package_directory.empty() && Configuration && !Configuration->ActiveSceneName.empty() && TrySceneDialogDirectory(Configuration->WorkingSpacePath.c_str(), Configuration->ScenePath.c_str(), {}, &directory)) { const std::filesystem::path source = std::filesystem::path(directory) / (std::string(Configuration->ActiveSceneName.c_str()) + ".zescene"); std::error_code error = {}; if (std::filesystem::is_regular_file(source, error) && !error) { const auto scene_path = SceneSourcePathFromNative(Configuration->WorkingSpacePath.c_str(), source.string()); - if (scene_path.Succeeded() && !RequestSceneOpen(scene_path.Value(), Session.SceneEpoch())) + if (scene_path.Succeeded() && m_source_capture && !m_source_capture_import_path.empty()) + { + m_source_capture_scene = scene_path.Value(); + } + else if (scene_path.Succeeded() && !RequestSceneOpen(scene_path.Value(), Session.SceneEpoch())) { ZENGINE_CORE_WARN("[Editor] Could not queue configured default scene {}", source.string()) } @@ -443,6 +503,29 @@ namespace Tetragrama } } + if (m_source_capture && !m_source_capture_import_path.empty() && engine && engine->ImportCoordinator) + { + const auto source = ZEngine::Core::VFS::VFSPath::Parse(m_source_capture_import_path.c_str()); + if (source.Succeeded()) + m_source_capture_import = engine->ImportCoordinator->Enqueue(source.Value()); + if (m_source_capture_import.is_nil() || !m_source_capture_scene.IsValid()) + { + ZENGINE_CORE_ERROR("[Editor] Cannot schedule source capture fixture import") + ZEngine::Engine::RequestClose(); + } + } + + if (!m_cooked_package_directory.empty() && engine) + { + m_cooked_initial_world = engine->Scene; + m_cooked_initial_render = CurrentScene; + m_cooked_initial_generation = engine->ScenePublication.Generation(); + m_cooked_startup = ZEngine::Helpers::Scope(new (std::nothrow) EditorCookedStartup{}); + m_cooked_startup_initialized = m_cooked_startup && m_cooked_startup->Initialize(*this, *engine, m_cooked_package_directory.c_str(), m_cooked_startup_scene); + if (!m_cooked_startup_initialized) + ZENGINE_CORE_ERROR("[Editor] Could not initialize cooked scene startup") + } + // Bake ZUI font atlases. Font sizes are chosen to be readable on high-resolution // displays where glfwGetWindowSize returns physical pixel counts (~3024px wide). // ImGui approach: fonts are baked at physical pixel density and all draw @@ -602,6 +685,301 @@ namespace Tetragrama void Editor::OnUpdate(float dt) { + if (m_source_capture && !m_source_capture_import.is_nil() && !m_source_capture_scene_queued) + { + auto* assets = ZEngine::Managers::AssetManager::Instance(); + bool ready = false; + bool failed = false; + if (assets && assets->Registry) + { + std::lock_guard lock(assets->IngestMutex); + const auto* record = assets->Registry->FindByUUID(m_source_capture_import); + ready = record && record->State == ZEngine::Core::VFS::AssetState::Loaded; + failed = record && record->State == ZEngine::Core::VFS::AssetState::Failed; + } + if (failed) + { + ZENGINE_CORE_ERROR("[Editor] Source capture fixture import failed") + ZEngine::Engine::RequestClose(); + } + if (ready) + { + m_source_capture_scene_queued = RequestSceneOpen(m_source_capture_scene, Session.SceneEpoch()); + if (!m_source_capture_scene_queued) + { + ZENGINE_CORE_ERROR("[Editor] Cannot queue source capture fixture scene") + ZEngine::Engine::RequestClose(); + } + } + } + if ((m_cooked_supersede_cpu_armed && m_cooked_startup && m_cooked_startup->CpuProbeReached()) || m_cooked_supersede_gpu_reached) + { + const bool gpu = m_cooked_supersede_gpu_reached; + auto* engine = ZEngine::Engine::GetContext(); + const auto old = m_cooked_reload_request; + ZEngine::ECS::SceneRequestId latest = {}; + ZEngine::ECS::SceneLoadResult superseded = {}; + if (!engine || !SceneOpenPending() || (!gpu && SceneOpenWorkPending()) || RequestCookedScene(m_cooked_startup_scene, Session.SceneEpoch(), &latest) != EditorCookedStartup::RestartResult::Started || !engine->SceneLoads.Poll(old, &superseded) || superseded.Outcome != ZEngine::ECS::SceneLoadOutcome::Superseded || !superseded.CleanupPending || !engine->SceneLoads.Acknowledge(old)) + { + m_cooked_startup_failed = true; + ZENGINE_CORE_ERROR("[Editor] Cooked CPU worker supersession failed") + } + else + { + m_cooked_reload_request = latest; + m_cooked_current_scene = m_cooked_startup_scene; + m_cooked_supersede_cpu_armed = false; + m_cooked_supersede_gpu_reached = false; + m_cooked_supersede_gpu = false; + if (gpu) + { + ZENGINE_CORE_INFO("[Editor] Superseded cooked GPU upload with cleanup pending") + } + else + { + ZENGINE_CORE_INFO("[Editor] Superseded cooked CPU worker with cleanup pending") + } + } + } + if (m_cooked_startup_initialized) + m_cooked_startup->Tick(*this); + if (m_cooked_staging_limit_active && (m_cooked_startup->Failed() || m_cooked_startup_failed)) + { + auto* engine = ZEngine::Engine::GetContext(); + const auto stats = engine->Device->GpuMem.GetAllocationStatistics(); + const auto domain = static_cast(ZEngine::Core::Memory::GpuMemoryDomain::HostStaging); + m_cooked_staging_refusal_observed = m_cooked_staging_gpu_started && m_cooked_staging_request_failed && stats.DomainBudgetRefusals[domain] > m_cooked_staging_refusal_baseline; + const bool restored = engine->Device->GpuMem.SetAllocationLimits(m_cooked_saved_gpu_limits); + ZENGINE_VALIDATE_ASSERT(restored, "Cooked test staging limit could not be restored") + m_cooked_staging_limit_active = false; + if (m_cooked_staging_refusal_observed) + ZENGINE_CORE_INFO("[Editor] Cooked replacement staging allocation was refused") + } + if (m_cooked_startup_exit_after_frame && !m_source_capture && (!m_cooked_startup_initialized || m_cooked_startup->Failed() || m_cooked_startup_failed)) + { + auto* engine = ZEngine::Engine::GetContext(); + ZEngine::Rendering::Scenes::SceneSubmissionProgress::Snapshot submitted = {}; + if (!engine || engine->Scene != m_cooked_initial_world || CurrentScene != m_cooked_initial_render || engine->ScenePublication.Generation() != m_cooked_initial_generation) + { + ZENGINE_CORE_ERROR("[Editor] Cooked startup failure changed the outgoing scene") + ZEngine::Engine::RequestClose(); + } + else if (RenderPipeline && RenderPipeline->SceneSubmissions().TryRead(&submitted)) + { + if (!m_cooked_failure_observed) + { + m_cooked_failure_observed = true; + m_cooked_failure_baseline = submitted.Generation == m_cooked_initial_generation ? submitted.Count : 0; + } + else if (submitted.Generation == m_cooked_initial_generation && submitted.Count > m_cooked_failure_baseline) + { + if (m_cooked_reload_request) + { + ZEngine::ECS::SceneLoadResult result = {}; + if (!engine->SceneLoads.Poll(m_cooked_reload_request, &result) || result.Outcome == ZEngine::ECS::SceneLoadOutcome::Pending || result.CleanupPending) + return; + if (!engine->SceneLoads.Acknowledge(m_cooked_reload_request)) + return; + m_cooked_reload_request = {}; + } + m_cooked_failure_preserved = true; + ZENGINE_CORE_INFO("[Editor] Outgoing scene submitted after cooked startup failure") + if (m_cooked_retry_after_failure && m_cooked_startup_initialized && (m_cooked_startup->Failed() || m_cooked_startup_failed)) + { + const auto retry = RequestCookedScene(m_cooked_startup_scene, Session.SceneEpoch(), &m_cooked_reload_request); + if (retry == EditorCookedStartup::RestartResult::Busy) + return; + if (retry == EditorCookedStartup::RestartResult::Started) + { + m_cooked_retry_after_failure = false; + m_cooked_startup_failed = false; + m_cooked_current_scene = m_cooked_startup_scene; + m_cooked_reload_generation = engine->ScenePublication.Generation() + 1; + ZENGINE_CORE_INFO("[Editor] Retrying cooked scene after preserved failure") + return; + } + } + ZEngine::Engine::RequestClose(); + } + } + } + if (m_cooked_startup_exit_after_frame && !m_cooked_startup_frame_recorded && m_loaded_scene) + { + ZEngine::Rendering::RenderResourceManager::SceneResourcePreparationResult result = {}; + ZEngine::Rendering::Scenes::SceneSubmissionProgress::Snapshot submitted = {}; + auto* engine = ZEngine::Engine::GetContext(); + // Observe multiple accepted frames: the prepared first frame and + // subsequent retained-resource frames use different render paths. + const bool awaiting_reload = !m_source_capture && m_cooked_reload_generation && (!engine || engine->ScenePublication.Generation() != m_cooked_reload_generation || SceneOpenPending() || m_cooked_startup->Pending()); + const bool prepared = !awaiting_reload && (m_source_capture ? !SceneOpenPending() : m_loaded_scene->PollRenderResources(&result) && result.Cooked && result.FrameRecorded); + if (engine && RenderPipeline && prepared && RenderPipeline->SceneSubmissions().TryRead(&submitted) && submitted.Generation == engine->ScenePublication.Generation() && submitted.Count >= 8) + { + if (m_cooked_reload_request) + { + ZEngine::ECS::SceneLoadResult outcome = {}; + if (!engine->SceneLoads.Poll(m_cooked_reload_request, &outcome) || outcome.Outcome != ZEngine::ECS::SceneLoadOutcome::Committed || outcome.CommittedGeneration != submitted.Generation || outcome.CleanupPending) + return; + if (!engine->SceneLoads.Acknowledge(m_cooked_reload_request)) + return; + m_cooked_reload_request = {}; + } + const auto* settings = m_loaded_scene->Document().Settings(); + if (!m_source_capture && (!settings || settings->UUID != m_cooked_current_scene)) + { + ZENGINE_CORE_ERROR("[Editor] Cooked scene replacement published the wrong scene identity") + ZEngine::Engine::RequestClose(); + return; + } + if (!m_source_capture && m_cooked_memory_sample_generation != submitted.Generation) + { + EditorCookedStartup::MemorySnapshot memory = {}; + if (m_cooked_startup->SampleMemory(&memory)) + { + const auto admitted = m_cooked_startup->AdmissionTimestampNs(); + if (!admitted || submitted.Generation != m_cooked_startup->ExpectedGeneration() || submitted.FirstTimestampNs < admitted) + { + ZENGINE_CORE_ERROR("[Editor] Invalid cooked admission-to-submission timing") + ZEngine::Engine::RequestClose(); + return; + } + ZENGINE_CORE_INFO("[CookedTiming] generation={} admission_ns={} first_submission_ns={} elapsed_ns={}", submitted.Generation, admitted, submitted.FirstTimestampNs, submitted.FirstTimestampNs - admitted) + const auto gpu = engine->Device->GpuMem.GetAllocationStatistics(); + const auto retirement = engine->Device->PendingFree.GetStatistics(); + ZENGINE_CORE_INFO("[CookedLifecycle] generation={} cpu_live={} cpu_peak={} staging_live={} staging_peak={} metadata_live={} metadata_blocks={} gpu_live={} gpu_peak={} retirement_queued={} retirement_draining={}", submitted.Generation, memory.CpuLive, memory.CpuPeak, memory.StagingLive, memory.StagingPeak, memory.MetadataLive, memory.MetadataBlocks, gpu.AllocationBytes, gpu.PeakAllocationBytes, retirement.Queued, retirement.Draining) + m_cooked_memory_sample_generation = submitted.Generation; + } + } + const uint32_t prepared_draws = m_source_capture ? m_loaded_scene->DrawCount() : result.InitialDrawCount; + const bool draws_match = !m_cooked_expected_draws || (prepared_draws == m_cooked_expected_draws && submitted.DrawCount == m_cooked_expected_draws); + if (!draws_match) + { + ZENGINE_CORE_ERROR("[Editor] Scene submitted {} draws (prepared {}, expected {})", submitted.DrawCount, prepared_draws, m_cooked_expected_draws) + ZEngine::Engine::RequestClose(); + } + else if (!m_source_capture && m_cooked_reloads_remaining) + { + if (m_cooked_fail_reload_staging && !m_cooked_staging_limit_active) + { + auto& allocator = engine->Device->GpuMem; + const auto stats = allocator.GetAllocationStatistics(); + const auto domain = static_cast(ZEngine::Core::Memory::GpuMemoryDomain::HostStaging); + auto limited = allocator.GetAllocationLimits(); + m_cooked_saved_gpu_limits = limited; + limited.DomainBytes[domain] = stats.DomainAdmittedBytes[domain]; + if (!allocator.SetAllocationLimits(limited)) + return; + m_cooked_staging_refusal_baseline = stats.DomainBudgetRefusals[domain]; + m_cooked_staging_limit_active = true; + } + const auto next = m_cooked_current_scene == m_cooked_startup_scene ? m_cooked_alternate_scene : m_cooked_startup_scene; + const auto restarted = RequestCookedScene(next, Session.SceneEpoch(), &m_cooked_reload_request); + if (restarted == EditorCookedStartup::RestartResult::Started) + { + if (m_cooked_reject_reload_submit) + { + engine->RenderResourceManager->ArmCookedBatchRejectionProbe(); + m_cooked_reject_reload_submit = false; + } + auto target = next; + if (m_cooked_supersede_reload) + { + const auto old = m_cooked_reload_request; + ZEngine::ECS::SceneRequestId latest = {}; + const auto newer = RequestCookedScene(m_cooked_startup_scene, Session.SceneEpoch(), &latest); + ZEngine::ECS::SceneLoadResult superseded = {}; + if (newer != EditorCookedStartup::RestartResult::Started || !SceneOpenPending() || SceneOpenWorkPending() || !engine->SceneLoads.Poll(old, &superseded) || superseded.Outcome != ZEngine::ECS::SceneLoadOutcome::Superseded || superseded.CleanupPending || !engine->SceneLoads.Acknowledge(old)) + { + m_cooked_startup_failed = true; + ZENGINE_CORE_ERROR("[Editor] Cooked waiting request supersession failed") + return; + } + m_cooked_reload_request = latest; + target = m_cooked_startup_scene; + m_cooked_supersede_reload = false; + ZENGINE_CORE_INFO("[Editor] Superseded cooked waiting request with a newer scene") + } + if (m_cooked_cancel_cpu) + m_cooked_startup->ArmCpuCancellationProbe(); + if (m_cooked_supersede_cpu) + { + m_cooked_startup->ArmCpuSupersessionProbe(); + m_cooked_supersede_cpu_armed = true; + m_cooked_supersede_cpu = false; + } + m_cooked_initial_world = engine->Scene; + m_cooked_initial_render = CurrentScene; + m_cooked_initial_generation = engine->ScenePublication.Generation(); + m_cooked_reload_generation = m_cooked_initial_generation + 1; + m_cooked_current_scene = target; + m_cooked_cancel_armed = m_cooked_cancel_after_mesh || m_cooked_supersede_gpu; + --m_cooked_reloads_remaining; + } + else if (restarted == EditorCookedStartup::RestartResult::Failed) + { + m_cooked_startup_failed = true; + ZENGINE_CORE_ERROR("[Editor] Cooked scene reload admission failed") + } + else if (restarted == EditorCookedStartup::RestartResult::Busy && m_cooked_staging_limit_active) + { + const bool restored = engine->Device->GpuMem.SetAllocationLimits(m_cooked_saved_gpu_limits); + ZENGINE_VALIDATE_ASSERT(restored, "Cooked test staging limit could not be restored") + m_cooked_staging_limit_active = false; + } + } + else if (m_cooked_capture_path.empty()) + { + m_cooked_startup_frame_recorded = true; + ZENGINE_CORE_INFO("[Editor] Cooked scene submitted {} frames with {} draws", submitted.Count, submitted.DrawCount) + ZEngine::Engine::RequestClose(); + } + else + { + const auto state = m_cooked_capture_state.load(std::memory_order_acquire); + if (state == 0) + { + ZENGINE_CORE_INFO("[SceneCapture] generation={} prepared_draws={} submitted_draws={}", submitted.Generation, prepared_draws, submitted.DrawCount) + ZEngine::Applications::RenderFrameState frame = {}; + if (!CaptureRenderView(frame) || !frame.RenderTargetW || !frame.RenderTargetH || uint64_t(frame.RenderTargetW) * frame.RenderTargetH > 32ULL * 1024 * 1024) + { + m_cooked_capture_state.store(4, std::memory_order_release); + } + else + { + m_cooked_capture_width = frame.RenderTargetW; + m_cooked_capture_height = frame.RenderTargetH; + m_cooked_capture_generation = submitted.Generation; + m_cooked_capture_bytes = size_t(frame.RenderTargetW) * frame.RenderTargetH * 4; + m_cooked_capture_pixels.reset(new (std::nothrow) uint8_t[m_cooked_capture_bytes]); + m_cooked_capture_state.store(m_cooked_capture_pixels ? 1 : 4, std::memory_order_release); + } + } + else if (state == 3) + { + const size_t count = size_t(m_cooked_capture_width) * m_cooked_capture_height; + auto* pixels = m_cooked_capture_pixels.get(); + for (size_t i = 0; i < count; ++i) + { + pixels[3 * i] = pixels[4 * i]; + pixels[3 * i + 1] = pixels[4 * i + 1]; + pixels[3 * i + 2] = pixels[4 * i + 2]; + } + std::ofstream image(m_cooked_capture_path, std::ios::binary | std::ios::trunc); + image << "P6\n" << m_cooked_capture_width << ' ' << m_cooked_capture_height << "\n255\n"; + image.write(reinterpret_cast(pixels), static_cast(count * 3)); + image.close(); + m_cooked_startup_frame_recorded = !image.fail(); + if (!m_cooked_startup_frame_recorded) + ZENGINE_CORE_ERROR("[Editor] Could not write cooked scene capture") + ZEngine::Engine::RequestClose(); + } + else if (state == 4) + { + ZENGINE_CORE_ERROR("[Editor] Cooked scene capture failed") + ZEngine::Engine::RequestClose(); + } + } + } + } auto* ctx = RenderPipeline ? RenderPipeline->ZUICtx : nullptr; auto* eng = ZEngine::Engine::GetContext(); const bool focused = (!eng || !eng->InputManager || eng->InputManager->IsWindowFocused()) && !m_focus_lost_pending; @@ -977,6 +1355,25 @@ namespace Tetragrama void Editor::OnRenderUI() { + if (m_cooked_runtime_view) + { + auto* ctx = RenderPipeline ? RenderPipeline->ZUICtx : nullptr; + if (!ctx || !RenderPipeline->SceneRenderer) + return; + if (State && ctx->ScreenW && ctx->ScreenH && (ctx->ScreenW != m_cooked_runtime_width || ctx->ScreenH != m_cooked_runtime_height) && State->RenderTargetResizeRequests.push({.Width = ctx->ScreenW, .Height = ctx->ScreenH})) + { + m_cooked_runtime_width = ctx->ScreenW; + m_cooked_runtime_height = ctx->ScreenH; + } + const auto texture = RenderPipeline->SceneRenderer->GetFrameOutput(); + auto* image = ZEngine::UI::ZUIBeginColumn(ctx, "##cooked_runtime", ZEngine::UI::ZPx(static_cast(ctx->ScreenW)), ZEngine::UI::ZPx(static_cast(ctx->ScreenH))); + image->Flags = image->Flags | ZEngine::UI::ZUI_DrawBackground; + image->TextureIndex = texture.Valid() ? static_cast(texture.Index) : 0xFFFFFFFFu; + image->EdgeSoftness = 0.f; + ZUIBoxSetColor(image, 1.f, 1.f, 1.f, texture.Valid() ? 1.f : 0.f); + ZEngine::UI::ZUIEndColumn(ctx); + return; + } if (ZUIUILayer) { ZUIUILayer->Render(nullptr, nullptr); @@ -992,6 +1389,15 @@ namespace Tetragrama { return; } + if (m_cooked_startup) + m_cooked_startup->Cancel(); + if (m_cooked_staging_limit_active) + { + auto* engine = ZEngine::Engine::GetContext(); + const bool restored = engine && engine->Device && engine->Device->GpuMem.SetAllocationLimits(m_cooked_saved_gpu_limits); + ZENGINE_VALIDATE_ASSERT(restored, "Cooked test staging limit could not be restored during shutdown") + m_cooked_staging_limit_active = false; + } DiscardSceneSettings(); if (ContentBrowser) ContentBrowser->CloseListings(); @@ -1023,14 +1429,53 @@ namespace Tetragrama } } - void Editor::OnClosed() {} + void Editor::OnClosed() + { + if (m_cooked_startup) + { + m_cooked_startup->AfterDeviceShutdown(); + m_cooked_startup.reset(); + } + m_cooked_capture_pixels.reset(); + } + + bool Editor::ApplyCookedRuntimeCamera(const ZEngine::ECS::Scene& scene, ZEngine::Applications::RenderFrameState& state) const + { + if (!m_cooked_runtime_view) + return true; + ZEngine::ECS::SceneDiagnostics diagnostics = {}; + if (ZEngine::ECS::CaptureSceneRuntimeCamera(scene, m_cooked_runtime_camera, state.RenderTargetW, state.RenderTargetH, &state.Camera, &diagnostics)) + return true; + for (uint32_t i = 0; i < diagnostics.Count; ++i) + ZENGINE_CORE_ERROR("[Editor] {}", diagnostics.Entries[i].Message) + return false; + } void Editor::PrepareScene(ZEngine::Applications::RenderFrameState& state) { GameApplication::PrepareScene(state); + if (m_cooked_runtime_view && m_loaded_scene && state.RenderTargetW && state.RenderTargetH && !ApplyCookedRuntimeCamera(m_loaded_scene->World(), state)) + { + m_cooked_startup_failed = true; + ZEngine::Engine::RequestClose(); + return; + } + state.SceneCapture = {}; + if (m_cooked_capture_state.load(std::memory_order_acquire) == 1) + { + if (m_cooked_runtime_view) + ZENGINE_CORE_INFO("[CookedRuntimeCamera] capture_position={} {} {}", state.Camera.Position.x, state.Camera.Position.y, state.Camera.Position.z) + state.SceneCapture = {this, &Editor::BeginCookedCapture, &Editor::CompleteCookedCapture, &Editor::CancelCookedCapture}; + } state.SceneResources = m_loaded_scene ? m_loaded_scene->InitialRenderResources() : ZEngine::Rendering::RenderResourceManager::SceneResourcePreparationId{}; UpdateCameraRevision(state.Camera); m_render_target_revision = state.ResizeSequence; + if (m_cooked_runtime_view) + { + state.ShowGrid = false; + state.Gizmo.Reset(); + return; + } PrepareSelectionOutline(state); PrepareObjectPick(state); PrepareHoverPick(state); diff --git a/Tetragrama/Editor.h b/Tetragrama/Editor.h index 16204fd1f..d047164d3 100644 --- a/Tetragrama/Editor.h +++ b/Tetragrama/Editor.h @@ -1,4 +1,6 @@ #pragma once +#include +#include #include #include #include @@ -75,6 +77,57 @@ namespace Tetragrama Editor(); ~Editor() override; + /// Configure before Initialize. A cooked launch uses no project source mount. + void ConfigureCookedStartup(std::string package_directory, const uuids::uuid& scene, bool exit_after_first_frame = false, uint32_t expected_draws = 0, std::string capture_path = {}, uint32_t reloads = 0, const uuids::uuid& alternate_scene = {}, bool cancel_reload_after_mesh = false, bool fail_reload_staging = false, bool cancel_reload_cpu = false); + /// Test-only source capture uses the same render-graph readback as cooked startup. + void ConfigureSourceCapture(std::string capture_path, uint32_t expected_draws, std::string import_path = {}); + /// The cooked authored-camera view uses a nil UUID to require exactly + /// one main camera in each candidate scene. + void ConfigureCookedRuntimeView(const uuids::uuid& camera) noexcept + { + m_cooked_runtime_view = true; + m_cooked_runtime_camera = camera; + } + void EnableCookedRetryProbe() noexcept + { + m_cooked_retry_after_failure = true; + } + void EnableCookedSupersedeProbe() noexcept + { + m_cooked_supersede_reload = true; + } + void EnableCookedCpuSupersedeProbe() noexcept + { + m_cooked_supersede_cpu = true; + } + void EnableCookedGpuSupersedeProbe() noexcept + { + m_cooked_supersede_gpu = true; + } + void EnableCookedSubmitRejectionProbe() noexcept + { + m_cooked_reject_reload_submit = true; + } + [[nodiscard]] bool CookedStartupFrameRecorded() const noexcept + { + return m_cooked_startup_frame_recorded; + } + [[nodiscard]] bool CookedStartupFailurePreserved() const noexcept + { + return m_cooked_failure_preserved; + } + [[nodiscard]] bool CookedReloadCancelledDuringUpload() const noexcept + { + return m_cooked_cancel_observed; + } + [[nodiscard]] bool CookedReloadCancelledDuringCpu() const noexcept + { + return m_cooked_startup && m_cooked_startup->CpuCancellationObserved(); + } + [[nodiscard]] bool CookedReloadStagingRefused() const noexcept + { + return m_cooked_staging_refusal_observed; + } Layers::ZUILayer* ZUIUILayer = nullptr; Panels::AssetImporterPanel* AssetImporter = nullptr; @@ -146,46 +199,56 @@ namespace Tetragrama { return false; } - [[nodiscard]] bool ProcessCloseRequest(ZEngine::EngineContext& engine); + [[nodiscard]] bool ProcessCloseRequest(ZEngine::EngineContext& engine); /// @brief Enqueue an atomic source save; unnamed scenes open the native Save As picker. - void SaveScene(); + void SaveScene(); /// @brief Opens the native save picker on the main thread, outside UI construction. - void SaveSceneAs(); - [[nodiscard]] bool RequestSceneSave(const ZEngine::Core::VFS::VFSPath& path, uint64_t expected_epoch); + void SaveSceneAs(); + [[nodiscard]] bool RequestSceneSave(const ZEngine::Core::VFS::VFSPath& path, uint64_t expected_epoch); /// @brief Called only within the session mutation phase, after queued edits. - [[nodiscard]] bool ProcessSceneSave(ZEngine::EngineContext& engine); + [[nodiscard]] bool ProcessSceneSave(ZEngine::EngineContext& engine); /// @brief Owner-thread polling; picker results retain the original scene/project binding. - void PollSceneSaveDialog(); + void PollSceneSaveDialog(); /// @brief Queued whole-scene reset. Dirty scenes are preserved, never silently discarded. - [[nodiscard]] bool RequestNewScene(); - [[nodiscard]] bool ProcessNewScene(ZEngine::EngineContext& engine); + [[nodiscard]] bool RequestNewScene(); + [[nodiscard]] bool ProcessNewScene(ZEngine::EngineContext& engine); /// Native dialog invocation is scheduled outside UI construction; completion /// is polled on the owner thread and retains the requesting scene/project binding. - void OpenSceneDialog(); - void PollSceneOpenDialog(); + void OpenSceneDialog(); + void PollSceneOpenDialog(); /// Queue a project-native source path on the session owner thread; never /// replaces the world inline. Dispatch captures project/service identities; /// stale work retires before preview cancellation or candidate publication. - void OpenScene(const char* path); + void OpenScene(const char* path); /// Optional output retains a pollable engine result until the caller /// acknowledges it. Without output, the editor releases it after cleanup. - [[nodiscard]] bool RequestSceneOpen(const ZEngine::Core::VFS::VFSPath& path, uint64_t expected_epoch, ZEngine::ECS::SceneRequestId* request = nullptr); - [[nodiscard]] bool ProcessSceneOpen(ZEngine::EngineContext& engine); - [[nodiscard]] bool SceneOpenPending() const; + [[nodiscard]] bool RequestSceneOpen(const ZEngine::Core::VFS::VFSPath& path, uint64_t expected_epoch, ZEngine::ECS::SceneRequestId* request = nullptr); + /// Request another scene from the mounted cooked package. Optional + /// result identity remains pollable through SceneLoads until acknowledged. + [[nodiscard]] EditorCookedStartup::RestartResult RequestCookedScene(const uuids::uuid& scene, uint64_t expected_epoch, ZEngine::ECS::SceneRequestId* request = nullptr); + /// Transfers a reconstructed candidate and ready closure only on admission. + /// The cache, routes and slab must outlive the committed scene and its + /// retired GPU resources, including after this request completes. + [[nodiscard]] bool RequestCookedSceneOpen(ZEngine::Helpers::Scope& candidate, ZEngine::Managers::CookedCpuClosureCache::Lease&& assets, uint64_t expected_epoch, ZEngine::Core::Memory::IAllocationRoute& route, ZEngine::Core::Memory::TLSFSlab& metadata, uint32_t max_textures, ZEngine::ECS::SceneRequestId* request = nullptr, ZEngine::ECS::SceneRequestId admitted_request = {}); + [[nodiscard]] bool ProcessSceneOpen(ZEngine::EngineContext& engine); + [[nodiscard]] bool SceneOpenPending() const; + /// Active editor handoff work, excluding the cooked CPU/waiting owner. + [[nodiscard]] bool SceneOpenWorkPending() const; /// Copies a drop intent with its original scene epoch; no I/O or authored write. - [[nodiscard]] bool RequestMeshDrop(const ZEngine::Core::VFS::VFSPath& path, uint64_t expected_epoch); + [[nodiscard]] bool RequestMeshDrop(const ZEngine::Core::VFS::VFSPath& path, uint64_t expected_epoch); /// Import completion queues placement; only the shared mutation-point loader creates entities. - [[nodiscard]] bool RequestImportedMesh(const ZEngine::EngineContext& engine, const ZEngine::Core::VFS::VFSPath& path, std::string_view name, const uuids::uuid& scene, uint64_t expected_epoch); + [[nodiscard]] bool RequestImportedMesh(const ZEngine::EngineContext& engine, const ZEngine::Core::VFS::VFSPath& path, std::string_view name, const uuids::uuid& scene, uint64_t expected_epoch); /// Complete, unpublished candidate only. Must run in the mutation phase; /// rejection preserves both the candidate and current committed content. /// Async callers supply the captured activation generation. Zero is only /// a synchronous caller's request to use the current generation. - [[nodiscard]] bool PublishSceneCandidate(ZEngine::EngineContext& engine, ZEngine::Helpers::Scope& candidate, uint64_t expected_epoch, uint64_t expected_generation = 0, ZEngine::ECS::SceneRequestId request = {}); + [[nodiscard]] bool PublishSceneCandidate(ZEngine::EngineContext& engine, ZEngine::Helpers::Scope& candidate, uint64_t expected_epoch, uint64_t expected_generation = 0, ZEngine::ECS::SceneRequestId request = {}); private: + [[nodiscard]] bool ApplyCookedRuntimeCamera(const ZEngine::ECS::Scene&, ZEngine::Applications::RenderFrameState&) const; ZEngine::Rendering::Renderers::GridLabelFontSnapshot CaptureGridLabelFont() const; enum class ObjectPickPhase : uint32_t @@ -315,6 +378,7 @@ namespace Tetragrama SceneFileOwner m_editor_close_owner = {}; bool m_editor_close_pending = false; struct SceneOpenWork; + struct CookedOpenWork; SceneFileRequest m_scene_open = {}; ZEngine::ECS::SceneRequestId m_scene_open_request = {}; ZEngine::EngineContext* m_scene_load_engine = nullptr; @@ -322,21 +386,77 @@ namespace Tetragrama ZEngine::Windows::FileDialogRequest m_scene_open_picker = {}; SceneFileOwner m_scene_open_picker_owner = {}; ZEngine::Helpers::Scope m_scene_open_work; - ZEngine::Helpers::Scope m_loaded_scene = {}; - ZEngine::ECS::Scene* m_original_world = nullptr; - ZEngine::ECS::ActorManager* m_original_actors = nullptr; - ZEngine::ECS::WorldCommands* m_original_commands = nullptr; - ZEngine::ECS::WorldTick* m_original_tick = nullptr; - bool m_publishing_scene = false; - bool m_processing_scene_open = false; - bool m_closing = false; + ZEngine::Helpers::Scope m_cooked_open_work; + ZEngine::Helpers::Scope m_cooked_startup; + std::string m_cooked_package_directory = {}; + uuids::uuid m_cooked_startup_scene = {}; + uuids::uuid m_cooked_alternate_scene = {}; + uuids::uuid m_cooked_current_scene = {}; + uuids::uuid m_cooked_runtime_camera = {}; + bool m_cooked_runtime_view = false; + uint32_t m_cooked_runtime_width = 0; + uint32_t m_cooked_runtime_height = 0; + uint32_t m_cooked_reloads_remaining = 0; + uint64_t m_cooked_reload_generation = 0; + ZEngine::ECS::SceneRequestId m_cooked_reload_request = {}; + bool m_cooked_cancel_after_mesh = false; + bool m_cooked_cancel_cpu = false; + bool m_cooked_retry_after_failure = false; + bool m_cooked_supersede_reload = false; + bool m_cooked_supersede_cpu = false; + bool m_cooked_supersede_cpu_armed = false; + bool m_cooked_supersede_gpu = false; + bool m_cooked_supersede_gpu_reached = false; + bool m_cooked_cancel_armed = false; + bool m_cooked_cancel_observed = false; + bool m_cooked_fail_reload_staging = false; + bool m_cooked_reject_reload_submit = false; + bool m_cooked_staging_limit_active = false; + bool m_cooked_staging_refusal_observed = false; + bool m_cooked_staging_gpu_started = false; + bool m_cooked_staging_request_failed = false; + uint64_t m_cooked_staging_refusal_baseline = 0; + ZEngine::Core::Memory::GpuAllocationLimits m_cooked_saved_gpu_limits = {}; + uint64_t m_cooked_memory_sample_generation = 0; + bool m_cooked_startup_initialized = false; + bool m_cooked_startup_exit_after_frame = false; + bool m_source_capture = false; + std::string m_source_capture_import_path = {}; + ZEngine::Core::VFS::VFSPath m_source_capture_scene = {}; + uuids::uuid m_source_capture_import = {}; + bool m_source_capture_scene_queued = false; + bool m_cooked_startup_frame_recorded = false; + bool m_cooked_startup_failed = false; + bool m_cooked_failure_observed = false; + bool m_cooked_failure_preserved = false; + // One bounded asynchronous viewport capture; callbacks run on render owner. + std::string m_cooked_capture_path = {}; + std::unique_ptr m_cooked_capture_pixels = {}; + std::atomic m_cooked_capture_state = 0; // 0 idle, 1 requested, 2 in flight, 3 complete, 4 failed + size_t m_cooked_capture_bytes = 0; + uint32_t m_cooked_capture_width = 0; + uint32_t m_cooked_capture_height = 0; + uint64_t m_cooked_capture_generation = 0; + uint64_t m_cooked_failure_baseline = 0; + uint64_t m_cooked_initial_generation = 0; + ZEngine::ECS::Scene* m_cooked_initial_world = nullptr; + ZEngine::Rendering::Scenes::RenderScene* m_cooked_initial_render = nullptr; + uint32_t m_cooked_expected_draws = 0; + ZEngine::Helpers::Scope m_loaded_scene = {}; + ZEngine::ECS::Scene* m_original_world = nullptr; + ZEngine::ECS::ActorManager* m_original_actors = nullptr; + ZEngine::ECS::WorldCommands* m_original_commands = nullptr; + ZEngine::ECS::WorldTick* m_original_tick = nullptr; + bool m_publishing_scene = false; + bool m_processing_scene_open = false; + bool m_closing = false; // A rapid native loss/regain still cancels at the next safe mutation point. - bool m_focus_lost_pending = false; - bool m_gizmo_focused = true; - EditorSceneChange m_scene_change = {}; - uint64_t m_scene_prompt_focus = 0; - ObjectPickFrame m_object_pick = {}; - ObjectPickFrame m_hover_pick = {}; + bool m_focus_lost_pending = false; + bool m_gizmo_focused = true; + EditorSceneChange m_scene_change = {}; + uint64_t m_scene_prompt_focus = 0; + ObjectPickFrame m_object_pick = {}; + ObjectPickFrame m_hover_pick = {}; static_assert(sizeof(ObjectPickFrame) * 2 <= 160 * 1024); ZEngine::Rendering::Cameras::CameraFrameData m_last_pick_camera = {}; uint64_t m_camera_revision = 1; @@ -348,6 +468,11 @@ namespace Tetragrama void CancelSceneOpen(); void ReleaseSceneOpenWaiting(); void ReleaseSceneOpenWork(); // Only after Ready or worker join. + [[nodiscard]] bool ProcessCookedSceneOpen(ZEngine::EngineContext& engine); + void ReleaseCookedSceneOpen(); + static bool BeginCookedCapture(void*, uint32_t, uint32_t, uint64_t); + static void CompleteCookedCapture(const void*, size_t, void*); + static void CancelCookedCapture(void*); }; ZDEFINE_PTR(Editor); diff --git a/Tetragrama/EditorCookedSceneRequest.cpp b/Tetragrama/EditorCookedSceneRequest.cpp new file mode 100644 index 000000000..d4c4c7b70 --- /dev/null +++ b/Tetragrama/EditorCookedSceneRequest.cpp @@ -0,0 +1,66 @@ +#include + +namespace Tetragrama +{ + EditorCookedSceneRequest::~EditorCookedSceneRequest() + { + const bool reset = Reset(); + ZENGINE_VALIDATE_ASSERT(reset, "Cooked scene request destroyed before cancellation completed") + (void) reset; + } + + bool EditorCookedSceneRequest::Begin(ZEngine::Helpers::Scope& candidate, ZEngine::Managers::CookedCpuClosureCache::Lease&& assets, ZEngine::ECS::SceneRequestId request, uint64_t generation, const ZEngine::Rendering::Scenes::SceneInitialView& view, ZEngine::Core::Memory::IAllocationRoute& route, ZEngine::Core::Memory::TLSFSlab& metadata, ZEngine::Rendering::RenderResourceManager& resources, ZEngine::Hardwares::VulkanDevice& device, ZEngine::Rendering::CookedSceneGpuMailbox& mailbox, uint32_t max_textures, bool pause_after_mesh_record) + { + if (!candidate || m_candidate || !m_request.Begin(candidate->Instance(), candidate->Render(), candidate->ResourcePreparation(), candidate->RequiredMeshes(), std::move(assets), request, generation, view, route, metadata, resources, device, mailbox, max_textures, pause_after_mesh_record)) + return false; + m_candidate = std::move(candidate); + return true; + } + + EditorCookedSceneRequest::State EditorCookedSceneRequest::Tick() + { + const auto state = m_request.Tick(); + if (state == State::Failed || state == State::Cancelled) + { + m_candidate.reset(); + m_request.ReleaseTerminalAssets(); + } + return state; + } + + void EditorCookedSceneRequest::Cancel() noexcept + { + m_request.Cancel(); + if (m_request.Status() == State::Cancelled) + { + m_candidate.reset(); + m_request.ReleaseTerminalAssets(); + } + } + + bool EditorCookedSceneRequest::Take(ZEngine::Helpers::Scope* output) + { + if (!output || *output || !m_request.TakeReady()) + return false; + *output = std::move(m_candidate); + return true; + } + + bool EditorCookedSceneRequest::Reset() noexcept + { + if (!m_request.Reset()) + return false; + m_candidate.reset(); + return true; + } + EditorCookedSceneRequest::State EditorCookedSceneRequest::Status() const noexcept + { + return m_request.Status(); + } + + bool EditorCookedSceneRequest::MeshUploadRecorded() const noexcept + { + return m_request.MeshUploadRecorded(); + } + +} // namespace Tetragrama diff --git a/Tetragrama/EditorCookedSceneRequest.h b/Tetragrama/EditorCookedSceneRequest.h new file mode 100644 index 000000000..7eeb8b255 --- /dev/null +++ b/Tetragrama/EditorCookedSceneRequest.h @@ -0,0 +1,30 @@ +#pragma once +#include +#include + +namespace Tetragrama +{ + /// Owns the editor candidate while the engine transaction borrows it. + class EditorCookedSceneRequest final + { + public: + using State = ZEngine::Rendering::Scenes::CookedSceneResourceRequest::State; + EditorCookedSceneRequest() = default; + ~EditorCookedSceneRequest(); + EditorCookedSceneRequest(const EditorCookedSceneRequest&) = delete; + EditorCookedSceneRequest& operator=(const EditorCookedSceneRequest&) = delete; + [[nodiscard]] bool Begin(ZEngine::Helpers::Scope&, ZEngine::Managers::CookedCpuClosureCache::Lease&&, ZEngine::ECS::SceneRequestId, uint64_t, const ZEngine::Rendering::Scenes::SceneInitialView&, ZEngine::Core::Memory::IAllocationRoute&, ZEngine::Core::Memory::TLSFSlab&, ZEngine::Rendering::RenderResourceManager&, ZEngine::Hardwares::VulkanDevice&, ZEngine::Rendering::CookedSceneGpuMailbox&, uint32_t, bool pause_after_mesh_record = false); + [[nodiscard]] State Tick(); + void Cancel() noexcept; + [[nodiscard]] bool Take(ZEngine::Helpers::Scope*); + [[nodiscard]] bool Reset() noexcept; + /// @brief Return the current resource request state. + [[nodiscard]] State Status() const noexcept; + /// @brief Report whether the mesh upload was recorded. + [[nodiscard]] bool MeshUploadRecorded() const noexcept; + + private: + ZEngine::Helpers::Scope m_candidate = {}; + ZEngine::Rendering::Scenes::CookedSceneResourceRequest m_request = {}; + }; +} // namespace Tetragrama diff --git a/Tetragrama/EditorCookedStartup.cpp b/Tetragrama/EditorCookedStartup.cpp new file mode 100644 index 000000000..55c59e461 --- /dev/null +++ b/Tetragrama/EditorCookedStartup.cpp @@ -0,0 +1,402 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Tetragrama +{ + namespace + { + // Explicit development limits, not a qualified runtime profile. + constexpr size_t CpuBytes = ZMega(1024ULL); + constexpr size_t GpuStagingBytes = ZMega(256ULL); + constexpr size_t MetadataBytes = ZMega(16ULL); + constexpr size_t ObjectBytes = ZMega(512ULL); + } // namespace + + bool EditorCookedStartup::Initialize(Editor& editor, ZEngine::EngineContext& engine, const char* directory, const uuids::uuid& scene) + { + if (!directory || !directory[0] || scene.is_nil() || !engine.SceneSchemas || !engine.SceneDocumentWorkingBytes || engine.SceneDocumentWorkingBytes > std::numeric_limits::max() - ZEngine::ECS::SceneStorageLease::MinimumBytes || !engine.SceneSchemas->FreezeForWorker(&m_diagnostics)) + return false; + if (m_cpu_allocator.Initialize(CpuBytes).Failed() || m_gpu_allocator.Initialize(GpuStagingBytes).Failed() || !m_metadata.TryInit(&editor.EditorArena, MetadataBytes)) + return false; + const ZEngine::Managers::CookedCpuAssetLimits limits = {.MaxAssets = 8192, .MaxObjectBytes = ObjectBytes, .MaxMeshWorkspaceBytes = ObjectBytes}; + if (!m_cache.Initialize(m_cpu_route, *engine.SceneSchemas, limits, 2, 32, 8192)) + return false; + m_backend.Initialize(directory, ZEngine::Core::VFS::VFSBackendCaps::Read | ZEngine::Core::VFS::VFSBackendCaps::List, &editor.EditorArena); + m_vfs.Initialize(&editor.EditorArena); + m_vfs_initialized = true; + if (m_vfs.Mount(&m_backend, ZEngine::Core::VFS::VFSPath::Root(), 0).Failed() || m_vfs.AcquireReadRoot(ZEngine::Core::VFS::VFSPath::Root(), &m_root).Failed()) + return false; + if (engine.AdmitSceneStorage(&m_storage, engine.SceneDocumentWorkingBytes + ZEngine::ECS::SceneStorageLease::MinimumBytes) != ZEngine::ECS::SceneStorageAdmission::Acquired) + return false; + m_engine = &engine; + m_scene = scene; + m_epoch = editor.Session.SceneEpoch(); + return AdmitRequest(); + } + + EditorCookedStartup::RestartResult EditorCookedStartup::Restart(Editor& editor, const uuids::uuid& scene, ZEngine::ECS::SceneRequestId* request) + { + if (!m_engine || m_engine->CloseRequested.value.load(std::memory_order_acquire) || !editor.Session.Ready() || scene.is_nil()) + return RestartResult::Failed; + const auto generation = m_engine->ScenePublication.Generation(); + const auto admitted_ns = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); + ZEngine::ECS::SceneRequestId admitted = {}; + if (!m_engine->SceneLoads.Admit(generation, &admitted)) + return RestartResult::Busy; + // Admit atomically supersedes older requests; their payloads remain + // owned until their worker or render work has actually drained. + ReleaseWaiting(); + m_waiting_request = admitted; + m_waiting_scene = scene; + m_waiting_epoch = editor.Session.SceneEpoch(); + m_waiting_generation = generation; + m_waiting_admission_ns = admitted_ns; + m_waiting_retain_result = request != nullptr; + m_waiting_pause_cpu = false; + m_waiting_supersession_probe = false; + if (request) + *request = admitted; + return RestartResult::Started; + } + + void EditorCookedStartup::ReleaseWaiting() noexcept + { + if (!m_waiting_request) + return; + auto& loads = m_engine->SceneLoads; + if (loads.Pending(m_waiting_request)) + loads.Cancel(m_waiting_request); + if (!m_waiting_retain_result) + loads.Acknowledge(m_waiting_request); + m_waiting_request = {}; + m_waiting_retain_result = false; + m_waiting_pause_cpu = false; + m_waiting_supersession_probe = false; + } + + void EditorCookedStartup::StartWaiting(Editor& editor) + { + if (!m_waiting_request) + return; + auto& loads = m_engine->SceneLoads; + if (!loads.Pending(m_waiting_request) || m_waiting_generation != m_engine->ScenePublication.Generation() || m_waiting_epoch != editor.Session.SceneEpoch() || m_engine->CloseRequested.value.load(std::memory_order_acquire)) + { + ReleaseWaiting(); + m_success = false; + return; + } + if (!m_done || m_request || editor.SceneOpenWorkPending() || m_candidate || m_reconstruction.Instance || m_reconstruction.Assets || m_root.IsValid() || m_storage.Arena()) + return; + const auto storage = m_engine->AdmitSceneStorage(&m_storage, m_engine->SceneDocumentWorkingBytes + ZEngine::ECS::SceneStorageLease::MinimumBytes); + if (storage == ZEngine::ECS::SceneStorageAdmission::Busy) + return; + if (storage != ZEngine::ECS::SceneStorageAdmission::Acquired) + { + loads.Fail(m_waiting_request, ZEngine::ECS::SceneLoadStage::Admission, ZEngine::ECS::SceneLoadError::Resources); + ReleaseWaiting(); + m_success = false; + return; + } + if (m_vfs.AcquireReadRoot(ZEngine::Core::VFS::VFSPath::Root(), &m_root).Failed()) + { + m_storage.Reset(); + loads.Fail(m_waiting_request, ZEngine::ECS::SceneLoadStage::Admission, ZEngine::ECS::SceneLoadError::Source); + ReleaseWaiting(); + m_success = false; + return; + } + if (!loads.TryStart(m_waiting_request)) + { + m_root.Reset(); + m_storage.Reset(); + return; + } + m_scene = m_waiting_scene; + m_epoch = m_waiting_epoch; + m_generation = m_waiting_generation; + m_admission_ns = m_waiting_admission_ns; + m_request = m_waiting_request; + m_retain_result = m_waiting_retain_result; + m_pause_cpu = m_waiting_pause_cpu; + m_cpu_supersession_probe = m_waiting_supersession_probe; + m_waiting_request = {}; + m_waiting_retain_result = false; + m_waiting_pause_cpu = false; + m_waiting_supersession_probe = false; + m_diagnostics.Clear(); + m_cancelled.value.store(false, std::memory_order_release); + m_failure_stage = ZEngine::ECS::SceneLoadStage::Read; + m_ready.value.store(false, std::memory_order_relaxed); + m_cache_cancel_token_ready.store(false, std::memory_order_relaxed); + m_cpu_boundary.store(false, std::memory_order_relaxed); + m_dispatched = false; + m_success = false; + m_done = false; + } + + bool EditorCookedStartup::AdmitRequest() noexcept + { + m_generation = m_engine->ScenePublication.Generation(); + m_admission_ns = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); + if (!m_engine->SceneLoads.Admit(m_generation, &m_request)) + return false; + if (m_engine->SceneLoads.TryStart(m_request)) + return true; + m_engine->SceneLoads.Fail(m_request, ZEngine::ECS::SceneLoadStage::Admission, ZEngine::ECS::SceneLoadError::Unavailable); + m_engine->SceneLoads.Acknowledge(m_request); + m_request = {}; + return false; + } + + void EditorCookedStartup::ReleaseRequest() noexcept + { + if (!m_request) + return; + auto& loads = m_engine->SceneLoads; + if (loads.Pending(m_request)) + loads.Cancel(m_request); + const bool cleaned = loads.FinishCleanup(m_request); + ZENGINE_VALIDATE_ASSERT(cleaned, "Cooked startup cleanup lost its admitted request") + (void) cleaned; + if (!m_retain_result) + loads.Acknowledge(m_request); + m_request = {}; + m_retain_result = false; + } + + bool EditorCookedStartup::SampleMemory(MemorySnapshot* output) const noexcept + { + if (!output || !m_done || !m_success || !m_ready.value.load(std::memory_order_acquire)) + return false; + const auto cpu = m_cpu_allocator.Statistics(); + const auto staging = m_gpu_allocator.Statistics(); + const auto metadata = m_metadata.GetStatistics(); + *output = {cpu.LiveBytes, cpu.PeakBytes, staging.LiveBytes, staging.PeakBytes, metadata.LiveBytes, metadata.LiveBlocks}; + return true; + } + + void EditorCookedStartup::Run(void* context) + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Worker.CookedRequest"); + auto& self = *static_cast(context); + const auto worker_begin = std::chrono::steady_clock::now(); + { + ZEngine::Managers::CookedPackageSnapshot package = {}; + ZEngine::Cook::CookDiagnostics diagnostics = {}; + const ZEngine::Cook::CookLimits limits = {.MaxWorkingBytes = CpuBytes}; + const auto captured = ZEngine::Managers::CookedPackageSnapshot::Capture(std::move(self.m_root), self.m_scene, *self.m_engine->SceneSchemas, limits, self.m_cpu_allocator, &diagnostics, &package, &self.m_cancelled.value); + const auto capture_end = std::chrono::steady_clock::now(); + if (captured.Failed()) + { + ZENGINE_CORE_ERROR("[Editor] Cooked package capture failed ({})", static_cast(captured.Error())) + for (uint32_t i = 0; i < diagnostics.Count; ++i) + ZENGINE_CORE_ERROR("[Editor] {}", diagnostics.Entries[i].Message) + } + if (captured.Succeeded() && !self.m_cancelled.value.load(std::memory_order_acquire)) + { + self.m_failure_stage = ZEngine::ECS::SceneLoadStage::Assets; + // Reclaim closures retired by earlier scenes before bounded cache + // admission. A cache hit can already be Ready and needs no pump. + while (self.m_cache.PumpOne()) + { + } + char path[ZEngine::Core::VFS::VFS_MAX_PATH] = {}; + const auto* scene = package.Scene(); + const int length = scene ? std::snprintf(path, sizeof(path), "/%s", scene->Source.CStr()) : -1; + const auto source = length > 0 && static_cast(length) < sizeof(path) ? ZEngine::Core::VFS::VFSPath::Parse(path) : ZEngine::Core::VFS::VFSResult::Fail(ZEngine::Core::VFS::VFSError::InvalidPath); + const auto reconstructed = source.Succeeded() ? ZEngine::ECS::ReconstructCookedScene(package, std::move(self.m_storage), *self.m_engine->SceneSchemas, self.m_cpu_route, ObjectBytes, self.m_engine->SceneDocumentWorkingBytes, self.m_cache, &self.m_reconstruction, &self.m_diagnostics, {}, &self.m_cancelled.value) : ZEngine::ECS::CookedSceneReconstructionError::InvalidInput; + const auto reconstruction_end = std::chrono::steady_clock::now(); + if (reconstructed != ZEngine::ECS::CookedSceneReconstructionError::None) + { + ZENGINE_CORE_ERROR("[Editor] Cooked scene reconstruction failed ({})", static_cast(reconstructed)) + } + else if (!self.m_cancelled.value.load(std::memory_order_acquire)) + { + self.m_cache_cancel_token = self.m_reconstruction.Assets.CancellationIdentity(); + self.m_cache_cancel_token_ready.store(true, std::memory_order_release); + if (self.m_pause_cpu) + { + self.m_cpu_boundary.store(true, std::memory_order_release); + while (!self.m_cancelled.value.load(std::memory_order_acquire)) + std::this_thread::yield(); + } + for (;;) + { + if (self.m_cancelled.value.load(std::memory_order_acquire)) + break; + const auto state = EditorSceneCandidate::PrepareReconstructedCooked(self.m_reconstruction, source.Value(), &self.m_candidate, &self.m_diagnostics); + if (state != CookedCandidateState::Pending) + { + self.m_success = state == CookedCandidateState::Ready; + break; + } + if (self.m_cancelled.value.load(std::memory_order_acquire) || !self.m_cache.PumpOne(&self.m_cancelled.value)) + break; + } + } + for (uint32_t i = 0; i < self.m_diagnostics.Count; ++i) + ZENGINE_CORE_ERROR("[Editor] {}", self.m_diagnostics.Entries[i].Message) + const auto preparation_end = std::chrono::steady_clock::now(); + const auto ns = [](auto duration) { return std::chrono::duration_cast(duration).count(); }; + ZENGINE_CORE_INFO("[CookedCpuTiming] capture_ns={} reconstruction_ns={} asset_preparation_ns={} worker_ns={}", ns(capture_end - worker_begin), ns(reconstruction_end - capture_end), ns(preparation_end - reconstruction_end), ns(preparation_end - worker_begin)) + } + } // Destroy any cache-hit package before publishing worker completion. + self.m_ready.value.store(true, std::memory_order_release); + } + + void EditorCookedStartup::Tick(Editor& editor) + { + if (m_done) + StartWaiting(editor); + if (m_done) + return; + ZENGINE_PROFILE_SCOPE("SceneLoad.Main.CookedStartup"); + if (m_pause_cpu && !m_cpu_supersession_probe && !m_cpu_cancel_observed && m_cpu_boundary.load(std::memory_order_acquire)) + { + ZEngine::ECS::SceneLoadResult result = {}; + if (m_engine->SceneLoads.Poll(m_request, &result) && result.CleanupPending && m_engine->SceneLoads.Cancel(m_request) == ZEngine::ECS::SceneCancelResult::Cancelled && m_engine->SceneLoads.Poll(m_request, &result) && result.CleanupPending) + { + m_cpu_cancel_observed = true; + ZENGINE_CORE_INFO("[Editor] Cancelled cooked replacement during CPU preparation with worker cleanup pending") + } + m_cancelled.value.store(true, std::memory_order_release); + } + if (!m_engine->SceneLoads.Pending(m_request) || m_generation != m_engine->ScenePublication.Generation() || m_epoch != editor.Session.SceneEpoch()) + m_cancelled.value.store(true, std::memory_order_release); + if (m_cancelled.value.load(std::memory_order_acquire)) + { + if (m_cache_cancel_token_ready.load(std::memory_order_acquire)) + (void) m_cache.CancelUnsharedBuild(m_cache_cancel_token); + if (m_dispatched && !m_ready.value.load(std::memory_order_acquire)) + return; + m_candidate.reset(); + m_reconstruction = {}; + m_storage.Reset(); + m_root.Reset(); + ReleaseRequest(); + m_done = true; + m_success = false; + return; + } + if (!m_dispatched) + { + auto* pool = ZEngine::Helpers::ThreadPoolHelper::Pool.get(); + if (pool && pool->Submit(this, &Run)) + m_dispatched = true; + return; + } + if (!m_ready.value.load(std::memory_order_acquire)) + return; + m_done = true; + const bool reconstructed = m_success; + ZEngine::ECS::SceneRequestId transferred = {}; + if (!reconstructed || !editor.RequestCookedSceneOpen(m_candidate, std::move(m_reconstruction.Assets), m_epoch, m_gpu_route, m_metadata, ZEngine::Rendering::RenderResourceManager::SceneCookedTextureCapacity, m_retain_result ? &transferred : nullptr, m_request)) + { + m_success = false; + m_engine->SceneLoads.Fail(m_request, reconstructed ? ZEngine::ECS::SceneLoadStage::Preparation : m_failure_stage, reconstructed ? ZEngine::ECS::SceneLoadError::Resources : ZEngine::ECS::SceneLoadError::Source); + m_candidate.reset(); + m_reconstruction = {}; + m_storage.Reset(); + m_root.Reset(); + ReleaseRequest(); + ZENGINE_CORE_ERROR("[Editor] Cooked scene startup failed; the current scene was preserved") + } + else + { + m_request = {}; // Cleanup ownership moved to CookedOpenWork. + m_retain_result = false; + } + } + + void EditorCookedStartup::Cancel() noexcept + { + m_cancelled.value.store(true, std::memory_order_release); + if (m_cache_cancel_token_ready.load(std::memory_order_acquire)) + (void) m_cache.CancelUnsharedBuild(m_cache_cancel_token); + } + + void EditorCookedStartup::AfterWorkersJoined() noexcept + { + m_candidate.reset(); + m_reconstruction = {}; + m_storage = {}; + m_root.Reset(); + ReleaseRequest(); + ReleaseWaiting(); + } + + void EditorCookedStartup::AfterDeviceShutdown() noexcept + { + while (m_cache.PumpOne()) + { + } + const bool closed = m_cache.Shutdown(); + ZENGINE_VALIDATE_ASSERT(closed, "Cooked startup cache still has retained scene resources") + (void) closed; + if (m_vfs_initialized) + m_vfs.Shutdown(); + if (m_metadata.Pool) + m_metadata.Shutdown(); + } + bool EditorCookedStartup::Pending() const noexcept + { + return !m_done || bool(m_waiting_request); + } + + bool EditorCookedStartup::OwnsEngine(const ZEngine::EngineContext* engine) const noexcept + { + return m_engine && m_engine == engine; + } + + uint64_t EditorCookedStartup::AdmissionTimestampNs() const noexcept + { + return m_admission_ns; + } + + uint64_t EditorCookedStartup::ExpectedGeneration() const noexcept + { + return m_generation + 1; + } + + void EditorCookedStartup::ArmCpuCancellationProbe() noexcept + { + if (m_waiting_request) + m_waiting_pause_cpu = true; + else + m_pause_cpu = true; + } + + void EditorCookedStartup::ArmCpuSupersessionProbe() noexcept + { + if (m_waiting_request) + { + m_waiting_pause_cpu = true; + m_waiting_supersession_probe = true; + } + } + + bool EditorCookedStartup::CpuProbeReached() const noexcept + { + return m_cpu_boundary.load(std::memory_order_acquire); + } + + bool EditorCookedStartup::CpuCancellationObserved() const noexcept + { + return m_cpu_cancel_observed; + } + + bool EditorCookedStartup::Failed() const noexcept + { + return m_done && !m_success && !m_waiting_request; + } + +} // namespace Tetragrama diff --git a/Tetragrama/EditorCookedStartup.h b/Tetragrama/EditorCookedStartup.h new file mode 100644 index 000000000..d741033cb --- /dev/null +++ b/Tetragrama/EditorCookedStartup.h @@ -0,0 +1,115 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Tetragrama +{ + struct Editor; + + // Process-lifetime startup owner. The package root is separate from the + // project VFS and its cache and routes remain alive until device teardown. + class EditorCookedStartup final + { + public: + enum class RestartResult : uint8_t + { + Started, + Busy, + Failed + }; + struct MemorySnapshot + { + uint64_t CpuLive = 0; + uint64_t CpuPeak = 0; + uint64_t StagingLive = 0; + uint64_t StagingPeak = 0; + size_t MetadataLive = 0; + size_t MetadataBlocks = 0; + }; + bool Initialize(Editor&, ZEngine::EngineContext&, const char* package_directory, const uuids::uuid& scene); + /// Called by the main owner after the previous request retired. The + /// scene/cache allocators remain shared with still-live scene pins. + [[nodiscard]] RestartResult Restart(Editor&, const uuids::uuid& scene, ZEngine::ECS::SceneRequestId* request = nullptr); + /// @brief Report whether startup or a replacement request remains pending. + [[nodiscard]] bool Pending() const noexcept; + /// @brief Report whether this owner belongs to the given engine. + [[nodiscard]] bool OwnsEngine(const ZEngine::EngineContext* engine) const noexcept; + /// Owner-thread sample only after the worker has completed and its + /// candidate has been transferred; it never races the CPU allocator. + [[nodiscard]] bool SampleMemory(MemorySnapshot* output) const noexcept; + /// @brief Return the request admission timestamp. + [[nodiscard]] uint64_t AdmissionTimestampNs() const noexcept; + /// @brief Return the generation assigned to the next scene. + [[nodiscard]] uint64_t ExpectedGeneration() const noexcept; + /// @brief Pause the current or waiting request at the CPU cancellation boundary. + void ArmCpuCancellationProbe() noexcept; + /// @brief Pause the waiting request for the CPU supersession probe. + void ArmCpuSupersessionProbe() noexcept; + /// @brief Report whether the CPU probe reached its boundary. + [[nodiscard]] bool CpuProbeReached() const noexcept; + /// @brief Report whether CPU cancellation was observed. + [[nodiscard]] bool CpuCancellationObserved() const noexcept; + void Tick(Editor&); + void Cancel() noexcept; + void AfterWorkersJoined() noexcept; + void AfterDeviceShutdown() noexcept; + /// @brief Report whether startup ended without a successful scene. + [[nodiscard]] bool Failed() const noexcept; + + private: + static void Run(void*); + bool AdmitRequest() noexcept; + void ReleaseRequest() noexcept; + void ReleaseWaiting() noexcept; + void StartWaiting(Editor&); + ZEngine::Cook::CookPageAllocator m_cpu_allocator = {}; + ZEngine::Cook::CookPageAllocator m_gpu_allocator = {}; + ZEngine::Cook::CookCodecAllocationRoute m_cpu_route{m_cpu_allocator}; + ZEngine::Cook::CookCodecAllocationRoute m_gpu_route{m_gpu_allocator}; + ZEngine::Core::Memory::TLSFSlab m_metadata = {}; + ZEngine::Core::VFS::VFSDiskBackend m_backend = {}; + ZEngine::Core::VFS::VFSContext m_vfs = {}; + ZEngine::Core::VFS::VFSReadRootLease m_root = {}; + ZEngine::Managers::CookedCpuClosureCache m_cache = {}; + ZEngine::ECS::SceneStorageLease m_storage = {}; + ZEngine::ECS::CookedSceneReconstruction m_reconstruction = {}; + ZEngine::Helpers::Scope m_candidate = {}; + ZEngine::ECS::SceneDiagnostics m_diagnostics = {}; + ZEngine::EngineContext* m_engine = nullptr; + uuids::uuid m_scene = {}; + uint64_t m_epoch = 0; + uint64_t m_generation = 0; + uint64_t m_admission_ns = 0; + ZEngine::ECS::SceneRequestId m_request = {}; + ZEngine::ECS::SceneRequestId m_waiting_request = {}; + uuids::uuid m_waiting_scene = {}; + uint64_t m_waiting_epoch = 0; + uint64_t m_waiting_generation = 0; + uint64_t m_waiting_admission_ns = 0; + ZEngine::Managers::CookedCpuClosureCache::CancellationToken m_cache_cancel_token = {}; + ZEngine::ECS::SceneLoadStage m_failure_stage = ZEngine::ECS::SceneLoadStage::Read; + /// @brief Isolate worker handoff flags from independently written state. + PaddedAtomic m_cancelled = {}; + PaddedAtomic m_ready = {}; + std::atomic m_cache_cancel_token_ready = false; + std::atomic m_cpu_boundary = false; + bool m_pause_cpu = false; + bool m_waiting_pause_cpu = false; + bool m_cpu_supersession_probe = false; + bool m_waiting_supersession_probe = false; + bool m_cpu_cancel_observed = false; + bool m_dispatched = false; + bool m_retain_result = false; + bool m_waiting_retain_result = false; + bool m_done = false; + bool m_success = false; + bool m_vfs_initialized = false; + }; +} // namespace Tetragrama diff --git a/Tetragrama/EditorSceneCandidate.cpp b/Tetragrama/EditorSceneCandidate.cpp index 722fc0521..c3faf51e5 100644 --- a/Tetragrama/EditorSceneCandidate.cpp +++ b/Tetragrama/EditorSceneCandidate.cpp @@ -4,14 +4,15 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include -#include using namespace ZEngine::ECS; using namespace ZEngine::ECS::Components; @@ -25,12 +26,6 @@ namespace Tetragrama SceneDiagnosticError(diagnostics, message); return false; } - struct MeshRequirement - { - uuids::uuid UUID = {}; - uint32_t Uses = 1; - }; - /// Retain the adapter's existing failure contract after the engine has /// reconstructed the ECS: later editor binding failure returns the input /// document before the unpublished engine owner releases its slot. @@ -51,79 +46,21 @@ namespace Tetragrama }; } // namespace - EditorSceneCandidate::~EditorSceneCandidate() - { - if (m_resources && m_resource_preparation) - { - m_resources->ReleaseSceneResourcePreparation(m_resource_preparation); - } - } + EditorSceneCandidate::~EditorSceneCandidate() = default; - bool EditorSceneCandidate::PrepareRenderResources(ZEngine::Rendering::RenderResourceManager& resources, SceneRequestId request, uint64_t generation, const ZEngine::Rendering::Scenes::SceneInitialView& view) + bool EditorSceneCandidate::PrepareRenderResources(ZEngine::Rendering::RenderResourceManager& resources, SceneRequestId request, uint64_t generation, const ZEngine::Rendering::Scenes::SceneInitialView& view, const ZEngine::Managers::CookedCpuAssetSet* cooked, bool* terminal_failure) { - ZENGINE_PROFILE_SCOPE("SceneLoad.Main.ResourceAdmission"); - if (m_resource_preparation) - { - if (m_resources != &resources) - { - return false; - } - if (m_resource_instances_copied < m_render.Instances.size()) - { - const auto count = std::min(size_t(64), m_render.Instances.size() - m_resource_instances_copied); - if (resources.AppendSceneResourceInstances(m_resource_preparation, {m_render.Instances.data() + m_resource_instances_copied, count})) - { - m_resource_instances_copied += static_cast(count); - } - // A refused batch is a terminal resource result, not backpressure. - // The caller polls it and preserves the outgoing generation. - } - return true; - } - // Resolve the candidate's sun, never the outgoing world's light state. - ZEngine::ECS::Systems::SyncHierarchy(World()); - // The unpublished adapter already has exact entity membership. Copy - // directly rather than performing a linear instance-ID lookup per entity. - for (auto& instance : m_render.Instances) - { - const auto* transform = World().GetComponent(instance.Entity); - if (!transform) - { - return false; - } - instance.Transform = transform->WorldTransform; - } - m_render.MarkInstancesDirty(); - ZEngine::ECS::Systems::SyncECSToLights(World(), m_render); - const ZEngine::Rendering::RenderResourceManager::SceneEnvironmentInput environment = {m_render.Sky, m_render.CelestialLight, m_render.SkyRevision.value.load(std::memory_order_acquire)}; - const ZEngine::Rendering::RenderResourceManager::SceneInitialFrameInput initial_frame = { - {m_render.Instances.data(), m_render.Instances.size()}, - m_render.PendingLights, m_render.Grid, view - }; - if (!resources.PrepareSceneResources(request, generation, {m_required_meshes.data(), m_required_meshes.size()}, &m_resource_preparation, ZEngine::Rendering::RenderResourceManager::ScenePreparationContent::MeshMaterials, &environment, &initial_frame, true)) - { - return false; - } - m_resource_instances_copied = 0; - m_resources = &resources; - return true; + return m_scene_resources.Prepare(*m_instance, m_render, {m_required_meshes.data(), m_required_meshes.size()}, resources, request, generation, view, cooked, terminal_failure); } bool EditorSceneCandidate::PollRenderResources(ZEngine::Rendering::RenderResourceManager::SceneResourcePreparationResult* out) const { - return m_resources && m_resources->PollSceneResourcePreparation(m_resource_preparation, out); + return m_scene_resources.Poll(out); } ZEngine::Rendering::RenderResourceManager::SceneResourcePreparationId EditorSceneCandidate::InitialRenderResources() { - ZEngine::Rendering::RenderResourceManager::SceneResourcePreparationResult result = {}; - if (PollRenderResources(&result) && result.FrameRecorded) - { - m_resources->ReleaseSceneResourcePreparation(m_resource_preparation); - m_resource_preparation = {}; - m_resources = nullptr; - } - return m_resource_preparation; + return m_scene_resources.InitialResources(); } bool EditorSceneCandidate::Prepare(ZEngine::EngineContext& engine, SceneDocument&& document, const SceneComponentSchemaRegistry& schemas, const ZEngine::Core::VFS::VFSPath& source, const EditorSceneAssetResolver& assets, ZEngine::Helpers::Scope* out, SceneDiagnostics* diagnostics, const EditorSceneCandidateLimits& limits) @@ -188,12 +125,73 @@ namespace Tetragrama return true; } - bool EditorSceneCandidate::PrepareRenderAdapter(const ZEngine::Core::VFS::VFSPath& source, const EditorSceneAssetResolver& assets, SceneDiagnostics* diagnostics, const EditorSceneCandidateLimits& limits) + bool EditorSceneCandidate::PreparePopulatedCooked(std::unique_ptr&& instance, const ZEngine::Core::VFS::VFSPath& source, const ZEngine::Managers::CookedCpuAssetSet& assets, ZEngine::Helpers::Scope* out, SceneDiagnostics* diagnostics, const EditorSceneCandidateLimits& limits) + { + if (!instance || !assets.IsValid() || !instance->MatchesCookedClosure(assets.ManifestDigest(), assets.SceneUUID())) + { + return Reject(diagnostics, "Scene binding requires the verified closure for this cooked scene and manifest."); + } + const auto* closure = &assets; + const EditorSceneAssetResolver resolver = {&closure, [](void* context, const uuids::uuid& uuid, uint32_t* draws, SceneDiagnostics*) { + const auto* set = *static_cast(context); + const ZEngine::Cook::CookAssetIdentity identity = {ZEngine::Cook::CookAssetKind::Mesh, uuid}; + const auto* mesh = set->FindMesh(identity); + if (!draws || !mesh || mesh->SubMeshes.empty() || mesh->SubMeshes.size() > UINT32_MAX) + { + return false; + } + *draws = static_cast(mesh->SubMeshes.size()); + return true; + }}; + if (!out || *out || !instance->Document().Settings() || instance->Actors().Count() || !instance->Commands().IsEmpty() || instance->Tick().SystemCount()) + { + return Reject(diagnostics, "Cooked editor binding requires an unpublished engine owner without actor lifecycle work."); + } + auto candidate = ZEngine::Helpers::Scope(new (std::nothrow) EditorSceneCandidate{}); + if (!candidate) + return Reject(diagnostics, "Cannot allocate editor scene ownership."); + candidate->m_instance = std::move(instance); + if (!candidate->PrepareRenderAdapter(source, resolver, diagnostics, limits, true)) + return false; + *out = std::move(candidate); + return true; + } + + CookedCandidateState EditorSceneCandidate::PrepareReconstructedCooked(CookedSceneReconstruction& reconstruction, const ZEngine::Core::VFS::VFSPath& source, ZEngine::Helpers::Scope* out, SceneDiagnostics* diagnostics, ZEngine::Core::VFS::VFSError* asset_error, const EditorSceneCandidateLimits& limits) + { + if (!out || *out || !reconstruction.Instance || !reconstruction.Assets) + { + Reject(diagnostics, "Cooked scene binding requires an unpublished reconstruction and an empty destination."); + return CookedCandidateState::Failed; + } + ZEngine::Managers::CookedCpuClosureCache::Result result = {}; + if (!reconstruction.Assets.Poll(&result)) + { + Reject(diagnostics, "The cooked asset lease is no longer valid."); + return CookedCandidateState::Failed; + } + if (result.StateValue == ZEngine::Managers::CookedCpuClosureCache::State::Pending) + { + return CookedCandidateState::Pending; + } + if (result.StateValue != ZEngine::Managers::CookedCpuClosureCache::State::Ready || !result.Assets) + { + if (asset_error) + { + *asset_error = result.Error; + } + Reject(diagnostics, "The cooked asset closure could not be prepared."); + return CookedCandidateState::Failed; + } + return PreparePopulatedCooked(std::move(reconstruction.Instance), source, *result.Assets, out, diagnostics, limits) ? CookedCandidateState::Ready : CookedCandidateState::Failed; + } + + bool EditorSceneCandidate::PrepareRenderAdapter(const ZEngine::Core::VFS::VFSPath& source, const EditorSceneAssetResolver& assets, SceneDiagnostics* diagnostics, const EditorSceneCandidateLimits& limits, bool cooked) { ZENGINE_PROFILE_SCOPE("SceneLoad.Worker.RenderAdapter"); constexpr auto gpu_budget = ZEngine::Rendering::Scenes::SceneData::MAX_DRAW_COMMANDS; const auto* settings = Document().Settings(); - if (!settings || !IsSceneSourcePath(source) || limits.MaxInstances > gpu_budget || limits.MaxDrawCommands > gpu_budget) + if (!settings || (cooked ? (!source.IsValid() || source.IsRoot()) : !IsSceneSourcePath(source)) || limits.MaxInstances > gpu_budget || limits.MaxDrawCommands > gpu_budget) { return Reject(diagnostics, "Invalid scene source or editor render limits."); } @@ -212,90 +210,24 @@ namespace Tetragrama { return Reject(diagnostics, "The legacy directional-light icon identity is invalid."); } - std::vector retired_icons = {}; - world.ForEach([&](EntityID entity, const MeshComponent& mesh, const LightComponent& light) { - if (mesh.MeshUUID == *legacy_icon && light.LightType == LightComponent::Type::Directional) - { - retired_icons.push_back(entity); - } - }); - for (const EntityID entity : retired_icons) { - world.RemoveComponent(entity); + ZEngine::Core::Memory::ArenaCheckpoint scratch(m_instance->Arena()); + ZEngine::Core::Containers::Array retired_icons = {}; + retired_icons.m_allocator = m_instance->Arena(); + bool scratch_ok = true; + world.ForEach([&](EntityID entity, const MeshComponent& mesh, const LightComponent& light) { + if (mesh.MeshUUID == *legacy_icon && light.LightType == LightComponent::Type::Directional && !retired_icons.try_push(entity)) + scratch_ok = false; + }); + if (!scratch_ok) + return Reject(diagnostics, "Scene storage cannot retain legacy icon cleanup scratch."); + for (const EntityID entity : retired_icons) + world.RemoveComponent(entity); + m_instance->RecordMemoryUsage(); } - // O(E + M log M), with one asset lookup per distinct mesh, not per placed - // instance. Temporary vectors exist only during an explicit scene load. - std::vector requirements = {}; - bool valid = true; - world.ForEach([&](EntityID entity, const MeshComponent& mesh) { - if (!world.HasComponent(entity) || mesh.MeshUUID.is_nil() || requirements.size() >= limits.MaxInstances) - { - valid = false; - return; - } - requirements.push_back({mesh.MeshUUID}); - }); - if (!valid) - { - return Reject(diagnostics, "A mesh has no transform/asset identity, or the complete scene exceeds the instance budget."); - } - const auto instance_count = static_cast(requirements.size()); - if (instance_count && !assets.ResolveMesh) - { - return Reject(diagnostics, "Scene meshes need an explicit asset resolver; no render bindings were published."); - } - std::sort(requirements.begin(), requirements.end(), [](const auto& a, const auto& b) { return a.UUID < b.UUID; }); - size_t unique_count = 0; - for (size_t i = 0; i < requirements.size(); ++i) - { - if (unique_count && requirements[unique_count - 1].UUID == requirements[i].UUID) - { - ++requirements[unique_count - 1].Uses; - } - else - { - requirements[unique_count++] = requirements[i]; - } - } - requirements.resize(unique_count); - m_required_meshes.m_allocator = m_instance->Arena(); - if (!m_required_meshes.try_reserve(unique_count)) - { - return Reject(diagnostics, "Cannot retain the complete scene resource identity set."); - } - for (const auto& requirement : requirements) - { - m_required_meshes.push(requirement.UUID); - uint32_t draws = 0; - if (!assets.ResolveMesh(assets.Context, requirement.UUID, &draws, diagnostics) || draws == 0) - { - return Reject(diagnostics, "A referenced mesh is missing or has no complete drawable binding; the active scene was preserved."); - } - const uint64_t additional = uint64_t(draws) * requirement.Uses; - if (additional > limits.MaxDrawCommands - m_draw_count) - { - return Reject(diagnostics, "The complete scene exceeds the GPU submesh draw budget; no instances were truncated."); - } - m_draw_count += static_cast(additional); - } - if (!render.Instances.try_reserve(instance_count)) - { - return Reject(diagnostics, "Cannot retain the complete scene instance list in its bounded arena."); - } - world.ForEach([&](EntityID entity, MeshComponent& mesh) { - ZEngine::Rendering::Scenes::MeshInstance instance = {}; - instance.Id = render.NextInstanceId++; - instance.MeshUUID = mesh.MeshUUID; - instance.Entity = entity; - instance.Transform = world.GetComponent(entity)->WorldTransform; - if (const auto* name = world.GetComponent(entity)) - { - ZEngine::Helpers::secure_strncpy(instance.Name, sizeof(instance.Name), name->Value, sizeof(instance.Name) - 1); - } - mesh.RenderInstanceId = instance.Id; - render.Instances.push(instance); - }); + if (!ZEngine::Rendering::Scenes::BuildSceneMeshBindings(*m_instance, render, assets, limits, m_required_meshes, m_draw_count, diagnostics)) + return false; render.Name = settings->Name.data(); // NUL-terminated text owned by the retained document. render.UUID = settings->UUID; @@ -303,7 +235,7 @@ namespace Tetragrama render.SyncGridFromAuthored(); render.Sky = settings->Sky; render.SourcePath = source; - render.SourceReadOnly = Document().ReadOnly(); + render.SourceReadOnly = cooked || Document().ReadOnly(); render.MarkDirty(false); render.MarkInstancesDirty(); render.MarkSkyDirty(); diff --git a/Tetragrama/EditorSceneCandidate.h b/Tetragrama/EditorSceneCandidate.h index fcd856573..b0334f295 100644 --- a/Tetragrama/EditorSceneCandidate.h +++ b/Tetragrama/EditorSceneCandidate.h @@ -3,23 +3,29 @@ #include #include #include +#include +#include #include +namespace ZEngine::Managers +{ + class CookedCpuAssetSet; +} +namespace ZEngine::ECS +{ + struct CookedSceneReconstruction; +} + namespace Tetragrama { - /// Resolves a complete CPU mesh binding, not GPU residency. Implementations may - /// populate the shared asset cache, but must not mutate the active scene or reset - /// global geometry. Return the complete number of submesh draws per instance. - struct EditorSceneAssetResolver - { - void* Context = nullptr; - bool (*ResolveMesh)(void* context, const uuids::uuid& uuid, uint32_t* out_draws, ZEngine::ECS::SceneDiagnostics* diagnostics) = nullptr; - }; + using EditorSceneAssetResolver = ZEngine::Rendering::Scenes::SceneMeshResolver; + using EditorSceneCandidateLimits = ZEngine::Rendering::Scenes::SceneMeshBindingLimits; - struct EditorSceneCandidateLimits + enum class CookedCandidateState : uint8_t { - uint32_t MaxInstances = ZEngine::Rendering::Scenes::SceneData::MAX_DRAW_COMMANDS; - uint32_t MaxDrawCommands = ZEngine::Rendering::Scenes::SceneData::MAX_DRAW_COMMANDS; + Pending, + Ready, + Failed }; /// Owns one unpublished or active loaded document in a reclaimable load slot. @@ -30,23 +36,41 @@ namespace Tetragrama { public: ~EditorSceneCandidate(); - EditorSceneCandidate(const EditorSceneCandidate&) = delete; - EditorSceneCandidate& operator=(const EditorSceneCandidate&) = delete; + EditorSceneCandidate(const EditorSceneCandidate&) = delete; + EditorSceneCandidate& operator=(const EditorSceneCandidate&) = delete; /// Explicit load-time work, never per-frame. Success consumes document and /// replaces out; failure preserves both. No active engine pointer is changed. - [[nodiscard]] static bool Prepare(ZEngine::EngineContext& engine, ZEngine::ECS::SceneDocument&& document, const ZEngine::ECS::SceneComponentSchemaRegistry& schemas, const ZEngine::Core::VFS::VFSPath& source, const EditorSceneAssetResolver& assets, ZEngine::Helpers::Scope* out, ZEngine::ECS::SceneDiagnostics* diagnostics, const EditorSceneCandidateLimits& limits = {}); + [[nodiscard]] static bool Prepare(ZEngine::EngineContext& engine, ZEngine::ECS::SceneDocument&& document, const ZEngine::ECS::SceneComponentSchemaRegistry& schemas, const ZEngine::Core::VFS::VFSPath& source, const EditorSceneAssetResolver& assets, ZEngine::Helpers::Scope* out, ZEngine::ECS::SceneDiagnostics* diagnostics, const EditorSceneCandidateLimits& limits = {}); /// Consumes an already admitted slot. No second reservation or fallback arena. - [[nodiscard]] static bool Prepare(ZEngine::ECS::SceneStorageLease&& storage, ZEngine::ECS::SceneDocument&& document, const ZEngine::ECS::SceneComponentSchemaRegistry& schemas, const ZEngine::Core::VFS::VFSPath& source, const EditorSceneAssetResolver& assets, ZEngine::Helpers::Scope* out, ZEngine::ECS::SceneDiagnostics* diagnostics, const EditorSceneCandidateLimits& limits = {}); + [[nodiscard]] static bool Prepare(ZEngine::ECS::SceneStorageLease&& storage, ZEngine::ECS::SceneDocument&& document, const ZEngine::ECS::SceneComponentSchemaRegistry& schemas, const ZEngine::Core::VFS::VFSPath& source, const EditorSceneAssetResolver& assets, ZEngine::Helpers::Scope* out, ZEngine::ECS::SceneDiagnostics* diagnostics, const EditorSceneCandidateLimits& limits = {}); /// Candidate-local editor policy on the exclusive reconstruction owner. /// Asset resolvers used on workers must be thread-safe. Consumes/discards the owner on binding failure; /// never repeats population, creates actors, or changes active bindings. - [[nodiscard]] static bool PreparePopulated(std::unique_ptr&& instance, const ZEngine::Core::VFS::VFSPath& source, const EditorSceneAssetResolver& assets, ZEngine::Helpers::Scope* out, ZEngine::ECS::SceneDiagnostics* diagnostics, const EditorSceneCandidateLimits& limits = {}); + [[nodiscard]] static bool PreparePopulated(std::unique_ptr&& instance, const ZEngine::Core::VFS::VFSPath& source, const EditorSceneAssetResolver& assets, ZEngine::Helpers::Scope* out, ZEngine::ECS::SceneDiagnostics* diagnostics, const EditorSceneCandidateLimits& limits = {}); + /// Resolve mesh draw counts from a verified cooked closure, without an + /// authoring registry. The closure must remain alive for this call; the + /// adapter retains only entity and asset identities, not CPU pointers. + [[nodiscard]] static bool PreparePopulatedCooked(std::unique_ptr&& instance, const ZEngine::Core::VFS::VFSPath& source, const ZEngine::Managers::CookedCpuAssetSet& assets, ZEngine::Helpers::Scope* out, ZEngine::ECS::SceneDiagnostics* diagnostics, const EditorSceneCandidateLimits& limits = {}); + /// Worker-only continuation of the cooked reconstruction transaction. + /// Pending and cache failure leave the instance and lease in the input; + /// Ready transfers the instance into the candidate and leaves the lease + /// with the caller for upload staging. Binding failure discards the + /// instance, as PreparePopulated does; output is unchanged on failure. + [[nodiscard]] static CookedCandidateState PrepareReconstructedCooked(ZEngine::ECS::CookedSceneReconstruction& reconstruction, const ZEngine::Core::VFS::VFSPath& source, ZEngine::Helpers::Scope* out, ZEngine::ECS::SceneDiagnostics* diagnostics, ZEngine::Core::VFS::VFSError* asset_error = nullptr, const EditorSceneCandidateLimits& limits = {}); - EditorScene& Render() + EditorScene& Render() { return m_render; } + ZEngine::ECS::SceneInstance& Instance() + { + return *m_instance; + } + ZEngine::Rendering::Scenes::SceneResourcePreparation& ResourcePreparation() + { + return m_scene_resources; + } ZEngine::ECS::Scene& World() { return m_instance->World(); @@ -71,26 +95,43 @@ namespace Tetragrama { return m_draw_count; } - bool PrepareRenderResources(ZEngine::Rendering::RenderResourceManager& resources, ZEngine::ECS::SceneRequestId request, uint64_t generation, const ZEngine::Rendering::Scenes::SceneInitialView& view = {}); + /// A false result is retryable unless terminal_failure is set. + bool PrepareRenderResources(ZEngine::Rendering::RenderResourceManager& resources, ZEngine::ECS::SceneRequestId request, uint64_t generation, const ZEngine::Rendering::Scenes::SceneInitialView& view = {}, const ZEngine::Managers::CookedCpuAssetSet* cooked = nullptr, bool* terminal_failure = nullptr); + /// Cooked candidates retain exact resource versions for their lifetime; + /// authoring candidates release their initial bundle after first record. ZEngine::Rendering::RenderResourceManager::SceneResourcePreparationId InitialRenderResources(); bool PollRenderResources(ZEngine::Rendering::RenderResourceManager::SceneResourcePreparationResult* out) const; size_t RequiredMeshCount() const { return m_required_meshes.size(); } + std::span RequiredMeshes() const + { + return {m_required_meshes.data(), m_required_meshes.size()}; + } + bool ResourceInstancesCollected() const noexcept + { + return m_scene_resources.InstancesCollected(m_render.Instances.size()); + } + bool HasRenderResourcePreparation() const noexcept + { + return m_scene_resources.HasPreparation(); + } + bool MatchesCookedClosure(const ZEngine::Core::Hash::Digest256& manifest, const uuids::uuid& scene) const noexcept + { + return m_instance && m_instance->MatchesCookedClosure(manifest, scene); + } private: EditorSceneCandidate() = default; - [[nodiscard]] bool PrepareRenderAdapter(const ZEngine::Core::VFS::VFSPath& source, const EditorSceneAssetResolver& assets, ZEngine::ECS::SceneDiagnostics* diagnostics, const EditorSceneCandidateLimits& limits); + [[nodiscard]] bool PrepareRenderAdapter(const ZEngine::Core::VFS::VFSPath& source, const EditorSceneAssetResolver& assets, ZEngine::ECS::SceneDiagnostics* diagnostics, const EditorSceneCandidateLimits& limits, bool cooked = false); // Destroy the borrowing editor adapter before the engine owner releases // actors/ECS/document and, last, its bounded scene-storage lease. - std::unique_ptr m_instance = {}; - EditorScene m_render = {}; - uint32_t m_draw_count = 0; - uint32_t m_resource_instances_copied = 0; - ZEngine::Core::Containers::Array m_required_meshes = {}; - ZEngine::Rendering::RenderResourceManager* m_resources = nullptr; - ZEngine::Rendering::RenderResourceManager::SceneResourcePreparationId m_resource_preparation = {}; + std::unique_ptr m_instance = {}; + EditorScene m_render = {}; + uint32_t m_draw_count = 0; + ZEngine::Core::Containers::Array m_required_meshes = {}; + ZEngine::Rendering::Scenes::SceneResourcePreparation m_scene_resources = {}; }; } // namespace Tetragrama diff --git a/Tetragrama/EditorSceneOpen.cpp b/Tetragrama/EditorSceneOpen.cpp index 6530fa6cb..9e81c789d 100644 --- a/Tetragrama/EditorSceneOpen.cpp +++ b/Tetragrama/EditorSceneOpen.cpp @@ -296,12 +296,50 @@ namespace Tetragrama } }; + struct Editor::CookedOpenWork + { + ZEngine::EngineContext* Engine = nullptr; + EditorScene* Scene = nullptr; + ZEngine::Applications::AppRenderPipeline* Pipeline = nullptr; + ZEngine::Rendering::RenderResourceManager* Resources = nullptr; + ZEngine::Hardwares::VulkanDevice* Device = nullptr; + ZEngine::Helpers::Scope Candidate = {}; + ZEngine::Managers::CookedCpuClosureCache::Lease Assets = {}; + ZEngine::Core::Memory::IAllocationRoute* Route = nullptr; + ZEngine::Core::Memory::TLSFSlab* Metadata = nullptr; + EditorCookedSceneRequest Preparation = {}; + ZEngine::ECS::SceneRequestId Request = {}; + uint64_t Epoch = 0; + uint64_t Generation = 0; + uint32_t MaxTextures = 0; + bool RetainResult = false; + }; + Editor::Editor() = default; Editor::~Editor() = default; + bool Editor::SceneOpenWorkPending() const + { + return m_scene_open.Pending || m_scene_open_work != nullptr || m_cooked_open_work != nullptr; + } + bool Editor::SceneOpenPending() const { - return m_scene_open.Pending || m_scene_open_work != nullptr; + return SceneOpenWorkPending() || (m_cooked_startup_initialized && m_cooked_startup && m_cooked_startup->Pending()); + } + + void Editor::ReleaseCookedSceneOpen() + { + if (!m_cooked_open_work) + return; + auto& loads = m_cooked_open_work->Engine->SceneLoads; + const auto request = m_cooked_open_work->Request; + const bool retain = m_cooked_open_work->RetainResult; + m_cooked_open_work.reset(); + const bool cleaned = loads.FinishCleanup(request); + ZENGINE_VALIDATE_ASSERT(cleaned, "Cooked scene cleanup lost its admitted request") + if (!retain) + loads.Acknowledge(request); } void Editor::ReleaseSceneOpenWaiting() @@ -347,6 +385,8 @@ namespace Tetragrama { m_scene_open_work->Cancelled.value.store(true, std::memory_order_release); } + if (m_cooked_open_work) + m_cooked_open_work->Preparation.Cancel(); // Retain the context even if the pool never started this queued task. // OnWorldShutdown releases it only after every worker has joined. } @@ -376,6 +416,14 @@ namespace Tetragrama AssetImporter->ShutdownAfterWorkersJoined(); } ReleaseSceneOpenWork(); // All workers have joined, including unstarted jobs. + if (m_cooked_open_work) + { + // Engine has drained the render mailbox after device idle. + (void) m_cooked_open_work->Preparation.Tick(); + ReleaseCookedSceneOpen(); + } + if (m_cooked_startup) + m_cooked_startup->AfterWorkersJoined(); m_scene_load_engine = nullptr; Session.Shutdown(); if (CurrentScene) @@ -517,7 +565,7 @@ namespace Tetragrama } SceneFileOwner snapshot = {}; auto* engine = owner->SessionEngine; - if (!engine || !engine->ScenePublication.Idle() || engine->CloseRequested.value.load(std::memory_order_acquire) || (SceneOpenPending() && m_scene_load_engine != engine) || !snapshot.Capture(*this)) + if (!engine || !engine->ScenePublication.Idle() || engine->CloseRequested.value.load(std::memory_order_acquire) || (SceneOpenWorkPending() && m_scene_load_engine != engine) || !snapshot.Capture(*this)) { return false; } @@ -547,6 +595,155 @@ namespace Tetragrama return true; } + EditorCookedStartup::RestartResult Editor::RequestCookedScene(const uuids::uuid& scene, uint64_t expected_epoch, ZEngine::ECS::SceneRequestId* request) + { + using Result = EditorCookedStartup::RestartResult; + if (!Session.OnOwnerThread() || m_closing || m_publishing_scene || m_processing_scene_open || m_editor_close_pending || m_new_scene_pending || SceneChangePromptState().Visible() || expected_epoch != Session.SceneEpoch() || !m_cooked_startup_initialized || !m_cooked_startup) + return Result::Failed; + const auto* owner = static_cast(CurrentScene); + if (!owner || owner->Session != &Session || !m_cooked_startup->OwnsEngine(owner->SessionEngine) || owner->SessionEngine->Scene != Session.Scene() || !owner->SessionEngine->ScenePublication.Idle()) + return Result::Failed; + return m_cooked_startup->Restart(*this, scene, request); + } + + bool Editor::RequestCookedSceneOpen(ZEngine::Helpers::Scope& candidate, ZEngine::Managers::CookedCpuClosureCache::Lease&& assets, uint64_t expected_epoch, ZEngine::Core::Memory::IAllocationRoute& route, ZEngine::Core::Memory::TLSFSlab& metadata, uint32_t max_textures, ZEngine::ECS::SceneRequestId* request, ZEngine::ECS::SceneRequestId admitted_request) + { + auto* owner = static_cast(CurrentScene); + auto* engine = owner ? owner->SessionEngine : nullptr; + ZEngine::Managers::CookedCpuClosureCache::Result decoded = {}; + if (!Session.OnOwnerThread() || m_closing || m_publishing_scene || m_processing_scene_open || SceneOpenWorkPending() || (!admitted_request && SceneOpenPending()) || m_editor_close_pending || m_new_scene_pending || SceneChangePromptState().Visible() || !owner || owner->Session != &Session || !Session.Ready() || expected_epoch != Session.SceneEpoch() || !engine || !engine->ScenePublication.Idle() || engine->CloseRequested.value.load(std::memory_order_acquire) || !RenderPipeline || !RenderPipeline->Device || !engine->RenderResourceManager || !candidate || + candidate->HasRenderResourcePreparation() || !assets.Poll(&decoded) || decoded.StateValue != ZEngine::Managers::CookedCpuClosureCache::State::Ready || !decoded.Assets || !candidate->MatchesCookedClosure(decoded.Assets->ManifestDigest(), decoded.Assets->SceneUUID()) || max_textures > ZEngine::Rendering::RenderResourceManager::SceneCookedTextureCapacity) + return false; + auto work = ZEngine::Helpers::Scope(new (std::nothrow) CookedOpenWork{}); + if (!work) + return false; + const auto generation = engine->ScenePublication.Generation(); + ZEngine::ECS::SceneRequestId admitted = admitted_request; + if (admitted) + { + ZEngine::ECS::SceneLoadResult result = {}; + if (!engine->SceneLoads.Poll(admitted, &result) || result.Outcome != ZEngine::ECS::SceneLoadOutcome::Pending || !result.CleanupPending || result.ExpectedGeneration != generation) + return false; + } + else if (!engine->SceneLoads.Admit(generation, &admitted)) + return false; + else if (!engine->SceneLoads.TryStart(admitted)) + { + engine->SceneLoads.Fail(admitted, ZEngine::ECS::SceneLoadStage::Admission, ZEngine::ECS::SceneLoadError::Unavailable); + engine->SceneLoads.Acknowledge(admitted); + return false; + } + work->Engine = engine; + work->Scene = owner; + work->Pipeline = RenderPipeline; + work->Resources = engine->RenderResourceManager; + work->Device = RenderPipeline->Device; + work->Candidate = std::move(candidate); + work->Assets = std::move(assets); + work->Route = &route; + work->Metadata = &metadata; + work->Request = admitted; + work->Epoch = expected_epoch; + work->Generation = generation; + work->MaxTextures = max_textures; + work->RetainResult = request != nullptr; + m_scene_load_engine = engine; + m_cooked_open_work = std::move(work); + if (request) + *request = admitted; + MeshDrop.Cancel(); + return true; + } + + bool Editor::ProcessCookedSceneOpen(ZEngine::EngineContext& engine) + { + auto& work = *m_cooked_open_work; + using State = EditorCookedSceneRequest::State; + const auto finish = [&](bool success) { + m_scene_change.Finish(SceneChangeKind::OpenScene, work.Epoch); + if (!success) + { + if (m_cooked_startup_exit_after_frame && !m_scene_open.Pending && (!m_cooked_startup || !m_cooked_startup->Pending())) + m_cooked_startup_failed = true; + work.Engine->SceneLoads.Fail(work.Request, ZEngine::ECS::SceneLoadStage::Preparation, ZEngine::ECS::SceneLoadError::Resources); + } + ReleaseCookedSceneOpen(); + return success; + }; + if (work.Engine != &engine || !engine.SceneLoads.Pending(work.Request) || m_closing || engine.CloseRequested.value.load(std::memory_order_acquire) || work.Generation != engine.ScenePublication.Generation() || work.Epoch != Session.SceneEpoch() || engine.Scene != Session.Scene() || CurrentScene != work.Scene || RenderPipeline != work.Pipeline || engine.RenderResourceManager != work.Resources || work.Pipeline->Device != work.Device) + { + work.Preparation.Cancel(); + const auto state = work.Preparation.Tick(); + if (state != State::Cancelling && state != State::GpuPending) + return finish(false); + return true; + } + if (Session.PendingRequestCount() || !ProcessSceneSettings(engine)) + return true; + Session.Cancel(); + const auto gate = GateSceneChange(SceneChangeKind::OpenScene); + if (gate == SceneChangeGate::Waiting) + return true; + if (gate != SceneChangeGate::Ready) + { + engine.SceneLoads.Cancel(work.Request); + work.Preparation.Cancel(); + const auto state = work.Preparation.Tick(); + if (state != State::Cancelling && state != State::GpuPending) + return finish(false); + return true; + } + if (work.Preparation.Status() == State::Empty) + { + ZEngine::Applications::RenderFrameState frame = {}; + if (!CaptureRenderView(frame) || !frame.RenderTargetW || !frame.RenderTargetH) + return true; + if (!ApplyCookedRuntimeCamera(work.Candidate->World(), frame)) + return finish(false); + if (m_cooked_runtime_view) + ZENGINE_CORE_INFO("[CookedRuntimeCamera] prepared_position={} {} {}", frame.Camera.Position.x, frame.Camera.Position.y, frame.Camera.Position.z) + ZEngine::Rendering::Scenes::SceneInitialView view = {frame.Camera, frame.RenderTargetW, frame.RenderTargetH, frame.ResizeSequence, !m_cooked_runtime_view && Session.GizmoSettings().ShowGrid, true}; + view.GridLabelFont = CaptureGridLabelFont(); + if (!work.Preparation.Begin(work.Candidate, std::move(work.Assets), work.Request, work.Generation, view, *work.Route, *work.Metadata, *engine.RenderResourceManager, *RenderPipeline->Device, RenderPipeline->CookedSceneUploads, work.MaxTextures, m_cooked_cancel_armed)) + return finish(false); + } + const auto state = work.Preparation.Tick(); + if (m_cooked_staging_limit_active && state == State::GpuPending) + m_cooked_staging_gpu_started = true; + if (m_cooked_staging_limit_active && state == State::Failed) + m_cooked_staging_request_failed = true; + if (m_cooked_cancel_armed && state == State::GpuPending && work.Preparation.MeshUploadRecorded()) + { + if (m_cooked_supersede_gpu) + { + m_cooked_supersede_gpu_reached = true; + m_cooked_cancel_armed = false; + return true; + } + if (engine.SceneLoads.Cancel(work.Request) != ZEngine::ECS::SceneCancelResult::Cancelled) + return finish(false); + work.Preparation.Cancel(); + m_cooked_cancel_armed = false; + m_cooked_cancel_observed = true; + ZENGINE_CORE_INFO("[Editor] Cancelled cooked replacement after mesh upload recording") + return true; + } + if (state == State::Failed || state == State::Cancelled) + { + ZENGINE_CORE_ERROR("[Editor] Cooked scene resource preparation failed") + return finish(false); + } + if (state != State::Ready) + return true; + if (!work.Preparation.Take(&work.Candidate) || !PublishSceneCandidate(engine, work.Candidate, work.Epoch, work.Generation, work.Request)) + { + ZENGINE_CORE_ERROR("[Editor] Cooked scene publication failed") + return finish(false); + } + ZENGINE_CORE_INFO("[Editor] Published cooked scene generation {}", engine.ScenePublication.Generation()) + return finish(true); + } + bool Editor::ProcessSceneOpen(ZEngine::EngineContext& engine) { ZENGINE_PROFILE_SCOPE("SceneLoad.Main.Process"); @@ -554,7 +751,7 @@ namespace Tetragrama { return false; } - if (!SceneOpenPending()) + if (!SceneOpenWorkPending()) { return true; } @@ -574,6 +771,8 @@ namespace Tetragrama Active = false; } } processing(m_processing_scene_open); + if (m_cooked_open_work) + return ProcessCookedSceneOpen(engine); // Terminal outcome can precede worker completion. Never free or replace // this payload until Ready, even if newer input is already waiting. if (m_scene_open_work) @@ -710,11 +909,24 @@ namespace Tetragrama { return true; // No drawable viewport yet; keep the outgoing scene. } - view = {captured.Camera, captured.RenderTargetW, captured.RenderTargetH, captured.ResizeSequence, Session.GizmoSettings().ShowGrid, true}; + if (m_cooked_runtime_view) + { + ZEngine::ECS::Systems::SyncHierarchy(work->Candidate->World()); + if (!ApplyCookedRuntimeCamera(work->Candidate->World(), captured)) + return finish(false, ZEngine::ECS::SceneLoadStage::Preparation, ZEngine::ECS::SceneLoadError::Resources); + } + view = {captured.Camera, captured.RenderTargetW, captured.RenderTargetH, captured.ResizeSequence, !m_cooked_runtime_view && Session.GizmoSettings().ShowGrid, true}; view.GridLabelFont = CaptureGridLabelFont(); } - if (!work->Candidate->PrepareRenderResources(*engine.RenderResourceManager, work->Request, work->Generation, view)) + bool terminal_failure = false; + if (!work->Candidate->PrepareRenderResources(*engine.RenderResourceManager, work->Request, work->Generation, view, nullptr, &terminal_failure)) { + if (terminal_failure) + { + work->Diagnostics.Error("The scene's render resources are invalid; the active scene was preserved."); + LogLoadDiagnostics(work->Diagnostics); + return finish(false, ZEngine::ECS::SceneLoadStage::Preparation, ZEngine::ECS::SceneLoadError::Resources); + } return true; // Both fixed request slots may still have mailbox/GPU owners. } ZEngine::Rendering::RenderResourceManager::SceneResourcePreparationResult resources = {}; @@ -905,6 +1117,8 @@ namespace Tetragrama { return false; } + if (!ApplyCookedRuntimeCamera(candidate->World(), frame)) + return false; // Preflight already synchronized hierarchy. An asynchronous request's // immutable resource input already captured exact transforms/lights; // do not repeat the authoring ID join on the publication frame. @@ -915,7 +1129,7 @@ namespace Tetragrama } frame.SceneGeneration = expected_generation + 1; frame.SceneResources = candidate->InitialRenderResources(); - frame.ShowGrid = Session.GizmoSettings().ShowGrid; + frame.ShowGrid = !m_cooked_runtime_view && Session.GizmoSettings().ShowGrid; frame.RenderOverlay = EnableRenderOverlay; frame.FramebufferW = CurrentWindow ? CurrentWindow->GetWidth() : 0; frame.FramebufferH = CurrentWindow ? CurrentWindow->GetHeight() : 0; diff --git a/Tetragrama/tests/CMakeLists.txt b/Tetragrama/tests/CMakeLists.txt index 801a4ab8e..432ff2190 100644 --- a/Tetragrama/tests/CMakeLists.txt +++ b/Tetragrama/tests/CMakeLists.txt @@ -12,6 +12,12 @@ target_sources(TetragramaTests PRIVATE ThumbnailService_test.cpp) target_sources(TetragramaTests PRIVATE DirectoryListing_test.cpp) target_link_libraries(TetragramaTests PRIVATE Tetragrama GTest::gtest_main) +# Reconstruction fixtures and their callees can exceed the Windows default +# thread stack. Reserve virtual address space; pages commit only as used. +if(MSVC) + target_link_options(TetragramaTests PRIVATE /STACK:8388608) +endif() + # Explicit real-editor smoke/soak runner. Not a CTest fixture: it requires a # desktop Vulkan session, packaged engine assets and a caller-owned project. add_executable(SceneLifecycleProcess EXCLUDE_FROM_ALL SceneLifecycleProcess.cpp) @@ -35,8 +41,10 @@ if(BUILD_TESTING) find_package(Python3 QUIET COMPONENTS Interpreter) if(Python3_Interpreter_FOUND) add_test(NAME SceneLifecycleEvidenceTests - COMMAND "${Python3_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/Scripts/tests/test_scene_lifecycle.py") + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/tests/scene/unit/test_scene_lifecycle.py") add_test(NAME SceneCaptureComparisonTests - COMMAND "${Python3_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/Scripts/tests/test_scene_capture.py") + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/tests/scene/unit/test_scene_capture.py") + add_test(NAME CookedRuntimeTraceEvidenceTests + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/tests/scene/unit/test_cooked_runtime_trace.py") endif() endif() diff --git a/Tetragrama/tests/EditorSceneCandidate_test.cpp b/Tetragrama/tests/EditorSceneCandidate_test.cpp index 68cd38c49..c53ce9d56 100644 --- a/Tetragrama/tests/EditorSceneCandidate_test.cpp +++ b/Tetragrama/tests/EditorSceneCandidate_test.cpp @@ -1,4 +1,14 @@ +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -6,11 +16,17 @@ #include #include #include +#include #include #include +#include +#include +#include #include +#include #include #include +#include "../../ZEngine/tests/Cook/CookTestAllocator.h" using namespace Tetragrama; using namespace ZEngine::Core::VFS; @@ -129,6 +145,39 @@ TEST_F(EditorSceneCandidateTest, RenderPreparationCopiesTheCandidatesResolvedSun EXPECT_EQ(result.Environment, ZEngine::Rendering::RenderResourceManager::SceneEnvironmentPreparationState::Queued); } +TEST_F(EditorSceneCandidateTest, RenderPreparationRejectsUnverifiedCookedClosureBeforeAdmission) +{ + AddEntity(7, true); + SceneDocument document = {}; + ASSERT_TRUE(Capture(&document)); + ASSERT_TRUE(Prepare(std::move(document))); + ResourceService = std::make_unique(); + ZEngine::Managers::CookedCpuAssetSet unverified = {}; + bool terminal_failure = false; + EXPECT_FALSE(Candidate->PrepareRenderResources(*ResourceService, {1, 1}, 1, {}, &unverified, &terminal_failure)); + EXPECT_TRUE(terminal_failure); + ZEngine::Rendering::RenderResourceManager::SceneResourcePreparationResult result = {}; + EXPECT_FALSE(Candidate->PollRenderResources(&result)); + EXPECT_TRUE(Candidate->PrepareRenderResources(*ResourceService, {1, 1}, 1)); + EXPECT_EQ(Candidate->RequiredMeshes().size(), 1u); +} + +TEST_F(EditorSceneCandidateTest, RenderPreparationReportsMissingInstanceTransformAsTerminal) +{ + AddEntity(7, true); + SceneDocument document = {}; + ASSERT_TRUE(Capture(&document)); + ASSERT_TRUE(Prepare(std::move(document))); + ASSERT_EQ(Candidate->Render().Instances.size(), 1u); + Candidate->World().RemoveComponent(Candidate->Render().Instances[0].Entity); + ResourceService = std::make_unique(); + bool terminal_failure = false; + EXPECT_FALSE(Candidate->PrepareRenderResources(*ResourceService, {1, 1}, 1, {}, nullptr, &terminal_failure)); + EXPECT_TRUE(terminal_failure); + ZEngine::Rendering::RenderResourceManager::SceneResourcePreparationResult result = {}; + EXPECT_FALSE(Candidate->PollRenderResources(&result)); +} + TEST_F(EditorSceneCandidateTest, AdmittedSlotTransfersToCandidateWithoutASecondReservation) { SceneDocument document = {}; @@ -181,6 +230,318 @@ TEST_F(EditorSceneCandidateTest, PopulatedAdapterCanBePreparedByItsExclusiveWork EXPECT_FALSE(AnySlotBusy()); } +TEST_F(EditorSceneCandidateTest, CookedAdapterRejectsUnverifiedClosureWithoutConsumingTheScene) +{ + AddEntity(7, true); + SceneDocument document = {}; + ASSERT_TRUE(Capture(&document)); + SceneStorageLease storage = {}; + ASSERT_TRUE(Engine->TryAcquireSceneStorage(&storage)); + std::unique_ptr instance = {}; + ASSERT_TRUE(SceneInstance::Create(std::move(storage), &instance, &Diagnostics)); + ASSERT_TRUE(instance->Populate(std::move(document), Schemas, &Diagnostics)); + ZEngine::Managers::CookedCpuAssetSet unverified = {}; + std::unique_ptr runtime_scene = {}; + EXPECT_FALSE(ZEngine::Rendering::Scenes::RuntimeScene::PrepareFromCooked(std::move(instance), unverified, &runtime_scene, &Diagnostics)); + EXPECT_NE(instance, nullptr); + EXPECT_EQ(runtime_scene, nullptr); + EXPECT_FALSE(EditorSceneCandidate::PreparePopulatedCooked(std::move(instance), Path, unverified, &Candidate, &Diagnostics)); + EXPECT_NE(instance, nullptr); + EXPECT_EQ(Candidate, nullptr); + EXPECT_EQ(Engine->Scene, &Source); +} + +TEST_F(EditorSceneCandidateTest, CookedReconstructionAdapterRejectsMissingWorkWithoutChangingOutput) +{ + ZEngine::ECS::CookedSceneReconstruction reconstruction = {}; + auto error = ZEngine::Core::VFS::VFSError::PermissionDenied; + EXPECT_EQ(EditorSceneCandidate::PrepareReconstructedCooked(reconstruction, Path, &Candidate, &Diagnostics, &error), CookedCandidateState::Failed); + EXPECT_EQ(Candidate, nullptr); + EXPECT_EQ(error, ZEngine::Core::VFS::VFSError::PermissionDenied); + EXPECT_EQ(Engine->Scene, &Source); +} + +TEST_F(EditorSceneCandidateTest, VerifiedCookedReconstructionWaitsForAssetsAndTransfersToCandidate) +{ + namespace Cook = ZEngine::Cook; + namespace VFS = ZEngine::Core::VFS; + using JSON = nlohmann::json; + std::printf("[CookedReconstructionTest] fixture setup\n"); + std::fflush(stdout); + AddEntity(7); + ASSERT_TRUE(Schemas.FreezeForWorker(&Diagnostics)); + SceneDocument source = {}; + ASSERT_TRUE(Capture(&source)); + ZEngine::Core::Containers::Array bytes = {}; + ASSERT_TRUE(BinarySceneCodec::Encode(source, Schemas, &Memory.MainArena, &bytes, &Diagnostics)); + auto allocator = std::make_unique>(); + Cook::CookCodecAllocationRoute route(*allocator); + VFS::VFSMemoryBackend backend = {}; + ASSERT_TRUE(backend.InitializeOwned(ZMega(1), 32).Succeeded()); + ASSERT_TRUE(backend.CreateDir(VFS::VFSPath::Parse("/package").Value()).Succeeded()); + ASSERT_TRUE(backend.CreateDir(VFS::VFSPath::Parse("/package/objects").Value()).Succeeded()); + VFS::VFSContext context = {}; + context.Initialize(&Memory.MainArena); + ASSERT_TRUE(context.Mount(&backend, VFS::VFSPath::Root(), 0).Succeeded()); + struct ContextGuard + { + VFS::VFSContext& Context; + ~ContextGuard() + { + Context.Shutdown(); + } + } context_guard{context}; + ZEngine::Core::Hash::SHA256 hash = {}; + Cook::Digest256 digest = {}; + ASSERT_TRUE(hash.Update({bytes.data(), bytes.size()}).Succeeded()); + ASSERT_TRUE(hash.Finalize(&digest).Succeeded()); + Cook::CookObjectDescriptor object = {}; + ASSERT_TRUE(Cook::MakeCookObjectDescriptor(digest, bytes.size(), Cook::CookSceneObjectExtension, &object).Succeeded()); + const auto object_path = std::string("/package/") + object.Path.CStr(); + ASSERT_TRUE(backend.WriteFile(VFS::VFSPath::Parse(object_path.c_str()).Value(), {bytes.data(), bytes.size()}).Succeeded()); + const auto words = Cook::CookSchemaKeyWorkspaceWords(Schemas); + ASSERT_TRUE(words.Succeeded()); + std::vector workspace(words.Value()); + Cook::Digest256 schema_key = {}; + ASSERT_TRUE(Cook::ComputeCookSchemaKey(*allocator, Schemas, {workspace.data(), workspace.size()}, &schema_key).Succeeded()); + const auto object_hex = digest.Hex(); + const auto schema_hex = schema_key.Hex(); + JSON manifest = { + {"manifest_version", 1}, + {"recipe_revision", 1}, + {"target", "desktop-vulkan-le64"}, + {"settings", JSON::object()}, + {"settings_key", "ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356"}, + {"schema_key", std::string(schema_hex.data(), 64)}, + {"assets", JSON::array()}, + {"scenes", JSON::array({{{"uuid", uuids::to_string(Settings.UUID)}, {"source", "scenes/loaded.zescene"}, {"source_digest", std::string(64, '1')}, {"input_key", std::string(64, '2')}, {"object", {{"path", object.Path.CStr()}, {"digest", std::string(object_hex.data(), 64)}, {"size", object.Size}}}, {"roots", JSON::array()}}})} + }; + const auto manifest_text = manifest.dump(); + ASSERT_TRUE(backend.WriteFile(VFS::VFSPath::Parse("/package/manifest.json").Value(), {reinterpret_cast(manifest_text.data()), manifest_text.size()}).Succeeded()); + VFS::VFSReadRootLease root = {}; + ASSERT_TRUE(context.AcquireReadRoot(VFS::VFSPath::Parse("/package").Value(), &root).Succeeded()); + ZEngine::Managers::CookedPackageSnapshot package = {}; + Cook::CookDiagnostics cook_diagnostics = {}; + const Cook::CookLimits cook_limits = {.MaxWorkingBytes = ZKilo(256)}; + std::printf("[CookedReconstructionTest] capture package\n"); + std::fflush(stdout); + const auto captured = ZEngine::Managers::CookedPackageSnapshot::Capture(std::move(root), Settings.UUID, Schemas, cook_limits, *allocator, &cook_diagnostics, &package); + ASSERT_TRUE(captured.Succeeded()) << "error " << static_cast(captured.Error()) << ", diagnostic " << (cook_diagnostics.Count ? cook_diagnostics.Entries[0].Message : "none"); + ZEngine::Managers::CookedCpuClosureCache cache = {}; + const ZEngine::Managers::CookedCpuAssetLimits limits = {.MaxAssets = 1, .MaxObjectBytes = ZKilo(64), .MaxMeshWorkspaceBytes = ZKilo(64)}; + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 1, 6, 1)); + struct CacheGuard + { + ZEngine::Managers::CookedCpuClosureCache& Cache; + ~CacheGuard() + { + while (Cache.PumpOne()) + { + } + EXPECT_TRUE(Cache.Shutdown()); + } + } cache_guard{cache}; + SceneStorageLease storage = {}; + ASSERT_TRUE(Engine->TryAcquireSceneStorage(&storage)); + CookedSceneReconstruction reconstruction = {}; + std::printf("[CookedReconstructionTest] reconstruct editor scene\n"); + std::fflush(stdout); + ASSERT_EQ(ReconstructCookedScene(package, std::move(storage), Schemas, route, ZKilo(64), ZMega(4), cache, &reconstruction, &Diagnostics), CookedSceneReconstructionError::None); + EXPECT_EQ(EditorSceneCandidate::PrepareReconstructedCooked(reconstruction, Path, &Candidate, &Diagnostics), CookedCandidateState::Pending); + ASSERT_NE(reconstruction.Instance, nullptr); + EXPECT_EQ(Candidate, nullptr); + { + // The same verified package can build an independent renderer-only + // scene owner without constructing an editor adapter or session. + std::printf("[CookedReconstructionTest] reconstruct standalone scene\n"); + std::fflush(stdout); + VFS::VFSReadRootLease runtime_root = {}; + ASSERT_TRUE(context.AcquireReadRoot(VFS::VFSPath::Parse("/package").Value(), &runtime_root).Succeeded()); + SceneStorageLease runtime_storage = {}; + ASSERT_TRUE(Engine->TryAcquireSceneStorage(&runtime_storage)); + ZEngine::Managers::CookedCpuClosureCache::Lease runtime_lease = {}; + std::unique_ptr runtime_scene = {}; + std::atomic cancelled = true; + using RuntimeReconstructionError = ZEngine::Applications::CookedRuntimeReconstructionError; + EXPECT_EQ(ZEngine::Applications::ReconstructCookedRuntimeScene(std::move(runtime_root), Settings.UUID, std::move(runtime_storage), Schemas, cook_limits, *allocator, route, ZKilo(64), ZMega(4), cache, &runtime_scene, &runtime_lease, &cook_diagnostics, &Diagnostics, &cancelled), RuntimeReconstructionError::Cancelled); + EXPECT_TRUE(runtime_root.IsValid()); + EXPECT_NE(runtime_storage.Arena(), nullptr); + EXPECT_EQ(runtime_scene, nullptr); + EXPECT_FALSE(runtime_lease); + cancelled.store(false); + ASSERT_EQ(ZEngine::Applications::ReconstructCookedRuntimeScene(std::move(runtime_root), Settings.UUID, std::move(runtime_storage), Schemas, cook_limits, *allocator, route, ZKilo(64), ZMega(4), cache, &runtime_scene, &runtime_lease, &cook_diagnostics, &Diagnostics, &cancelled), RuntimeReconstructionError::None); + ZEngine::Managers::CookedCpuClosureCache::Result runtime_assets = {}; + ASSERT_TRUE(runtime_lease.Poll(&runtime_assets)); + ASSERT_EQ(runtime_assets.StateValue, ZEngine::Managers::CookedCpuClosureCache::State::Ready); + ASSERT_NE(runtime_assets.Assets, nullptr); + ASSERT_NE(runtime_scene, nullptr); + EXPECT_EQ(runtime_scene->Instance().World().AliveCount(), 1u); + EXPECT_EQ(runtime_scene->DrawCount(), 0u); + EXPECT_EQ(runtime_scene->Render().Instances.size(), 0u); + EXPECT_EQ(Engine->Scene, &Source); + auto runtime_resources = std::make_unique(); + auto runtime_device = std::make_unique(); + runtime_device->RRM = runtime_resources.get(); + auto runtime_app = std::make_unique(); + auto runtime_pipeline = std::make_unique(); + runtime_app->RenderPipeline = runtime_pipeline.get(); + Engine->RenderResourceManager = runtime_resources.get(); + ZEngine::Applications::RuntimeSceneHost runtime_host = {}; + ZEngine::Applications::RenderFrameState runtime_frame = {}; + runtime_frame.RenderTargetW = 1280; + runtime_frame.RenderTargetH = 720; + EXPECT_FALSE(runtime_host.Publish(*Engine, *runtime_app, runtime_scene, {5, 1}, Engine->ScenePublication.Generation(), {}, runtime_frame, &Diagnostics)); + ASSERT_NE(runtime_scene, nullptr); + EXPECT_EQ(Engine->Scene, &Source); + EXPECT_EQ(Engine->ScenePublication.Generation(), 1u); + EXPECT_EQ(runtime_app->CurrentScene, nullptr); + ZEngine::Core::Memory::TLSFSlab runtime_metadata = {}; + runtime_metadata.Init(&Memory.MainArena, ZKilo(16)); + ZEngine::Rendering::CookedSceneGpuMailbox runtime_mailbox = {}; + ZEngine::Rendering::Scenes::CookedSceneResourceRequest runtime_request = {}; + using RuntimeRequest = ZEngine::Rendering::Scenes::CookedSceneResourceRequest; + std::printf("[CookedReconstructionTest] prepare standalone resources\n"); + std::fflush(stdout); + ASSERT_TRUE(runtime_request.Begin(runtime_scene->Instance(), runtime_scene->Render(), runtime_scene->Resources(), runtime_scene->RequiredMeshes(), std::move(runtime_lease), {5, 1}, 1, {}, route, runtime_metadata, *runtime_resources, *runtime_device, runtime_mailbox, 0)); + ASSERT_EQ(runtime_request.Tick(), RuntimeRequest::State::GpuPending); + for (uint32_t i = 0; i < 8 && runtime_request.Status() == RuntimeRequest::State::GpuPending; ++i) + { + ASSERT_TRUE(runtime_mailbox.PumpOne(0, 0)); + (void) runtime_request.Tick(); + } + ASSERT_EQ(runtime_request.Status(), RuntimeRequest::State::ResourcePending); + runtime_request.Cancel(); + ASSERT_EQ(runtime_request.Status(), RuntimeRequest::State::Cancelled); + runtime_scene.reset(); + runtime_request.ReleaseTerminalAssets(); + EXPECT_TRUE(runtime_request.Reset()); + Engine->RenderResourceManager = nullptr; + } + std::printf("[CookedReconstructionTest] prepare editor resources\n"); + std::fflush(stdout); + ASSERT_EQ(EditorSceneCandidate::PrepareReconstructedCooked(reconstruction, Path, &Candidate, &Diagnostics), CookedCandidateState::Ready); + EXPECT_EQ(reconstruction.Instance, nullptr); + EXPECT_TRUE(reconstruction.Assets); + ASSERT_NE(Candidate, nullptr); + EXPECT_EQ(Candidate->World().AliveCount(), 1u); + EXPECT_EQ(Engine->Scene, &Source); + ZEngine::Managers::CookedCpuClosureCache::Result assets = {}; + ASSERT_TRUE(reconstruction.Assets.Poll(&assets)); + ASSERT_NE(assets.Assets, nullptr); + ResourceService = std::make_unique(); + auto device = std::make_unique(); + device->RRM = ResourceService.get(); + ZEngine::Core::Memory::TLSFSlab metadata = {}; + metadata.Init(&Memory.MainArena, ZKilo(16)); + ZEngine::Rendering::CookedSceneGpuMailbox mailbox = {}; + EditorCookedSceneRequest request = {}; + ASSERT_TRUE(request.Begin(Candidate, std::move(reconstruction.Assets), {4, 1}, 1, {}, route, metadata, *ResourceService, *device, mailbox, 0)); + EXPECT_EQ(Candidate, nullptr); + ASSERT_EQ(request.Tick(), EditorCookedSceneRequest::State::GpuPending); + for (uint32_t i = 0; i < 8 && request.Status() == EditorCookedSceneRequest::State::GpuPending; ++i) + { + ASSERT_TRUE(mailbox.PumpOne(0, 0)); + (void) request.Tick(); + } + ASSERT_EQ(request.Status(), EditorCookedSceneRequest::State::ResourcePending); + EXPECT_TRUE(mailbox.Drained()); + request.Cancel(); + EXPECT_EQ(request.Status(), EditorCookedSceneRequest::State::Cancelled); + EXPECT_TRUE(request.Reset()); + ASSERT_TRUE(cache.PumpOne()); + + std::printf("[CookedReconstructionTest] exercise request ledger\n"); + std::fflush(stdout); + // Exercise the editor's actual request ledger and cancellation before a + // drawable viewport exists. No authoring VFS or asset registry is used. + VFS::VFSReadRootLease second_root = {}; + ASSERT_TRUE(context.AcquireReadRoot(VFS::VFSPath::Parse("/package").Value(), &second_root).Succeeded()); + ZEngine::Managers::CookedPackageSnapshot second_package = {}; + ASSERT_TRUE(ZEngine::Managers::CookedPackageSnapshot::Capture(std::move(second_root), Settings.UUID, Schemas, cook_limits, *allocator, &cook_diagnostics, &second_package).Succeeded()); + ASSERT_TRUE(Engine->TryAcquireSceneStorage(&storage)); + CookedSceneReconstruction second = {}; + ASSERT_EQ(ReconstructCookedScene(second_package, std::move(storage), Schemas, route, ZKilo(64), ZMega(4), cache, &second, &Diagnostics), CookedSceneReconstructionError::None); + ASSERT_TRUE(cache.PumpOne()); + const auto cooked_source = VFS::VFSPath::Parse("/scenes/fixture.scene").Value(); + ASSERT_EQ(EditorSceneCandidate::PrepareReconstructedCooked(second, cooked_source, &Candidate, &Diagnostics), CookedCandidateState::Ready); + EXPECT_TRUE(Candidate->Render().SourceReadOnly); + { + // These owners contain large fixed-capacity buffers; keep them off the + // test thread's stack, which is smaller on Windows. + auto editor_owner = std::make_unique(); + auto outgoing_owner = std::make_unique(); + auto pipeline_owner = std::make_unique(); + auto& editor = *editor_owner; + auto& outgoing = *outgoing_owner; + auto& pipeline = *pipeline_owner; + pipeline.Device = device.get(); + editor.RenderPipeline = &pipeline; + editor.CurrentScene = &outgoing; + Engine->RenderResourceManager = ResourceService.get(); + ASSERT_TRUE(editor.Session.Initialize(&Memory.MainArena, {.MaxTargets = 8, .HistoryEntries = 8, .HistoryBytes = 4096})); + ASSERT_TRUE(editor.Session.BeginMutationPhase()); + ASSERT_TRUE(editor.Session.BindScene(&Source)); + ASSERT_TRUE(outgoing.AttachSession(editor.Session, *Engine)); + editor.Session.EndMutationPhase(); + SceneRequestId id = {}; + ASSERT_TRUE(editor.RequestCookedSceneOpen(Candidate, std::move(second.Assets), editor.Session.SceneEpoch(), route, metadata, 0, &id)); + EXPECT_EQ(Candidate, nullptr); + EXPECT_TRUE(editor.SceneOpenPending()); + EXPECT_EQ(Engine->Scene, &Source); + EXPECT_EQ(Engine->SceneLoads.Cancel(id), SceneCancelResult::Cancelled); + ASSERT_TRUE(editor.Session.BeginMutationPhase()); + EXPECT_FALSE(editor.ProcessSceneOpen(*Engine)); + editor.Session.EndMutationPhase(); + EXPECT_FALSE(editor.SceneOpenPending()); + SceneLoadResult result = {}; + ASSERT_TRUE(Engine->SceneLoads.Poll(id, &result)); + EXPECT_EQ(result.Outcome, SceneLoadOutcome::Cancelled); + EXPECT_FALSE(result.CleanupPending); + EXPECT_TRUE(Engine->SceneLoads.Acknowledge(id)); + + // A source request can supersede a cooked candidate. The new request + // waits until the old candidate's cleanup releases its coordinator slot. + while (cache.PumpOne()) + { + } + VFS::VFSReadRootLease third_root = {}; + ASSERT_TRUE(context.AcquireReadRoot(VFS::VFSPath::Parse("/package").Value(), &third_root).Succeeded()); + ZEngine::Managers::CookedPackageSnapshot third_package = {}; + ASSERT_TRUE(ZEngine::Managers::CookedPackageSnapshot::Capture(std::move(third_root), Settings.UUID, Schemas, cook_limits, *allocator, &cook_diagnostics, &third_package).Succeeded()); + ASSERT_TRUE(Engine->TryAcquireSceneStorage(&storage)); + CookedSceneReconstruction third = {}; + ASSERT_EQ(ReconstructCookedScene(third_package, std::move(storage), Schemas, route, ZKilo(64), ZMega(4), cache, &third, &Diagnostics), CookedSceneReconstructionError::None); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_EQ(EditorSceneCandidate::PrepareReconstructedCooked(third, cooked_source, &Candidate, &Diagnostics), CookedCandidateState::Ready); + SceneRequestId cooked = {}, source_request = {}; + ASSERT_TRUE(editor.RequestCookedSceneOpen(Candidate, std::move(third.Assets), editor.Session.SceneEpoch(), route, metadata, 0, &cooked)); + ASSERT_TRUE(editor.RequestSceneOpen(Path, editor.Session.SceneEpoch(), &source_request)); + ASSERT_TRUE(Engine->SceneLoads.Poll(cooked, &result)); + EXPECT_EQ(result.Outcome, SceneLoadOutcome::Superseded); + EXPECT_TRUE(result.CleanupPending); + EXPECT_FALSE(Engine->SceneLoads.TryStart(source_request)); + ASSERT_TRUE(editor.Session.BeginMutationPhase()); + EXPECT_FALSE(editor.ProcessSceneOpen(*Engine)); + editor.Session.EndMutationPhase(); + ASSERT_TRUE(Engine->SceneLoads.Poll(cooked, &result)); + EXPECT_EQ(result.Outcome, SceneLoadOutcome::Superseded); + EXPECT_FALSE(result.CleanupPending); + EXPECT_EQ(Engine->Scene, &Source); + EXPECT_TRUE(Engine->SceneLoads.Acknowledge(cooked)); + EXPECT_EQ(Engine->SceneLoads.Cancel(source_request), SceneCancelResult::Cancelled); + ASSERT_TRUE(editor.Session.BeginMutationPhase()); + EXPECT_TRUE(editor.ProcessSceneOpen(*Engine)); + editor.Session.EndMutationPhase(); + EXPECT_FALSE(editor.SceneOpenPending()); + EXPECT_TRUE(Engine->SceneLoads.Acknowledge(source_request)); + editor.OnWorldShutdown(*Engine); + editor.CurrentScene = nullptr; + } + metadata.Shutdown(); + ASSERT_TRUE(cache.PumpOne()); + EXPECT_TRUE(cache.Shutdown()); +} + TEST_F(EditorSceneCandidateTest, EmptyDocumentHasOwnedMetadataAndNoDefaultContent) { SceneDocument document = {}; diff --git a/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp b/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp index ce5ffb0eb..a7539da8c 100644 --- a/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp +++ b/ZEngine/ZEngine/Applications/AppRenderPipeline.cpp @@ -265,11 +265,17 @@ namespace ZEngine::Applications void AppRenderPipeline::RenderScene(const RenderFrameState& state, const Rendering::Renderers::ZUIRenderPayload* overlay) { ZENGINE_PROFILE_SCOPE("SceneLoad.Render.RecordScene"); - const auto& scene = state.Scene; - auto swpachain = Device->SwapchainPtr; - auto frame_index = swpachain->CurrentFrame->Index; - auto thread_index = RenderMainThreadIndex; - auto* rrm = Device->RRM ? static_cast(Device->RRM) : nullptr; + const auto& scene = state.Scene; + auto swpachain = Device->SwapchainPtr; + auto frame_index = swpachain->CurrentFrame->Index; + auto thread_index = RenderMainThreadIndex; + auto* rrm = Device->RRM ? static_cast(Device->RRM) : nullptr; + Rendering::RenderResourceManager::SceneResourcePreparationResult retained_resources = {}; + if (state.SceneResources && (!rrm || !rrm->PollSceneResourcePreparation(state.SceneResources, &retained_resources))) + { + return; + } + const bool cooked_resources = state.SceneResources && retained_resources.Cooked; Rendering::Scenes::PreparedSceneFrame initial_frame = {}; const bool use_initial_frame = rrm && state.SceneResources && rrm->GetPreparedSceneFrame(state.SceneResources, &initial_frame); if (use_initial_frame && !rrm->GetFramePreparedSceneFrame(state.SceneResources, &initial_frame)) @@ -384,6 +390,11 @@ namespace ZEngine::Applications } else { + if (cooked_resources) + { + prepared_frame_complete = false; + continue; + } // Ordinary frames, including assets added by authoring after // activation, keep the existing streaming path. Initial // prepared meshes never resolve their newer public handles. @@ -553,6 +564,10 @@ namespace ZEngine::Applications hover_pick.Submitted(hover_pick.Context); } SceneRenderer->BeginSkyFrame(camera); + if (prepared_frame_complete && state.SceneCapture.Context && state.SceneCapture.Begin && state.SceneCapture.Complete && state.SceneCapture.Cancelled && state.SceneCapture.Begin(state.SceneCapture.Context, state.RenderTargetW, state.RenderTargetH, state.SceneGeneration)) + { + SceneRenderer->RequestSceneColorCapture(state.SceneCapture.Complete, state.SceneCapture.Cancelled, state.SceneCapture.Context); + } auto* const recorded = SceneRenderer->DrawScene(frame_index, thread_index, CurrentCmdBuf, camera, use_initial_frame ? &initial_frame : nullptr); if (recorded) { @@ -573,11 +588,12 @@ namespace ZEngine::Applications ZUIRenderPass->SetPayload(nullptr); } - void AppRenderPipeline::OnSceneFrameSubmitted(void* context, Rendering::Primitives::Semaphore*, uint64_t) + void AppRenderPipeline::OnSceneFrameSubmitted(void* context, Rendering::Primitives::Semaphore* timeline, uint64_t timeline_value) { - auto& owner = *static_cast(context); - const auto now = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); - owner.m_scene_submissions.Record(owner.m_recorded_scene_generation, static_cast(now), owner.m_recorded_scene_draw_count); + auto& owner = *static_cast(context); + const auto now = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + const bool expected_timeline = owner.Device && owner.Device->SwapchainPtr && owner.Device->SwapchainPtr->RenderTimeline == timeline; + owner.m_scene_submissions.Record(owner.m_recorded_scene_generation, static_cast(now), owner.m_recorded_scene_draw_count, expected_timeline ? timeline_value : 0); } void AppRenderPipeline::BeginOverlayFrame(float dt) diff --git a/ZEngine/ZEngine/Applications/AppRenderPipeline.h b/ZEngine/ZEngine/Applications/AppRenderPipeline.h index 7413834be..a54684b4b 100644 --- a/ZEngine/ZEngine/Applications/AppRenderPipeline.h +++ b/ZEngine/ZEngine/Applications/AppRenderPipeline.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include #include @@ -41,6 +42,14 @@ namespace ZEngine::Applications } }; + struct RenderSceneCaptureRequest + { + void* Context = nullptr; + bool (*Begin)(void*, uint32_t, uint32_t, uint64_t) = nullptr; + Rendering::Renderers::RGReadbackFn Complete = nullptr; + Rendering::Renderers::RGReadbackCancelFn Cancelled = nullptr; + }; + /// @brief Ownership state for a producer/consumer payload slot. enum class PayloadSlotState : uint32_t { @@ -76,6 +85,7 @@ namespace ZEngine::Applications /// Hover has its own immutable token table and completion context. It /// must never consume or resolve a click-selection request. RenderObjectPickRequest HoverPick = {}; + RenderSceneCaptureRequest SceneCapture = {}; // Physical window size, captured on the main thread with platform events. uint32_t FramebufferW = 0; uint32_t FramebufferH = 0; @@ -92,6 +102,8 @@ namespace ZEngine::Applications struct AppRenderPipeline { + /// Producer-to-render handoff for unpublished cooked scene uploads. + Rendering::CookedSceneGpuMailbox CookedSceneUploads = {}; /// Main-producer-thread reservation. The pipeline must outlive this token. /// Destruction cancels an unpublished frame; Publish only releases an /// already captured slot and cannot allocate, copy a scene or fail. diff --git a/ZEngine/ZEngine/Applications/CookedRuntimeReconstruction.cpp b/ZEngine/ZEngine/Applications/CookedRuntimeReconstruction.cpp new file mode 100644 index 000000000..97ad2bd47 --- /dev/null +++ b/ZEngine/ZEngine/Applications/CookedRuntimeReconstruction.cpp @@ -0,0 +1,67 @@ +#include +#include +#include + +namespace ZEngine::Applications +{ + CookedRuntimeReconstructionError ReconstructCookedRuntimeScene( + Core::VFS::VFSReadRootLease&& root, + const uuids::uuid& scene, + ECS::SceneStorageLease&& storage, + const ECS::SceneComponentSchemaRegistry& schemas, + const Cook::CookLimits& limits, + Cook::ICookAllocator& allocator, + Core::Memory::IAllocationRoute& route, + size_t max_scene_bytes, + size_t document_working_bytes, + Managers::CookedCpuClosureCache& cache, + std::unique_ptr* output, + Managers::CookedCpuClosureCache::Lease* output_assets, + Cook::CookDiagnostics* cook_diagnostics, + ECS::SceneDiagnostics* scene_diagnostics, + const std::atomic* cancelled) + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Worker.RuntimeReconstruction"); + const auto is_cancelled = [&] { return cancelled && cancelled->load(std::memory_order_acquire); }; + if (!root.IsValid() || scene.is_nil() || !storage.Arena() || !output || *output || !output_assets || *output_assets || !max_scene_bytes || !document_working_bytes) + return CookedRuntimeReconstructionError::InvalidInput; + if (is_cancelled()) + return CookedRuntimeReconstructionError::Cancelled; + Managers::CookedPackageSnapshot package = {}; + const auto captured = Managers::CookedPackageSnapshot::Capture(std::move(root), scene, schemas, limits, allocator, cook_diagnostics, &package, cancelled); + if (captured.Failed()) + return is_cancelled() ? CookedRuntimeReconstructionError::Cancelled : CookedRuntimeReconstructionError::Package; + ECS::CookedSceneReconstruction reconstruction = {}; + const auto reconstructed = ECS::ReconstructCookedScene(package, std::move(storage), schemas, route, max_scene_bytes, document_working_bytes, cache, &reconstruction, scene_diagnostics, {}, cancelled); + if (reconstructed != ECS::CookedSceneReconstructionError::None) + return is_cancelled() ? CookedRuntimeReconstructionError::Cancelled : CookedRuntimeReconstructionError::Reconstruction; + for (;;) + { + if (is_cancelled()) + { + (void) cache.CancelUnsharedBuild(reconstruction.Assets.CancellationIdentity()); + return CookedRuntimeReconstructionError::Cancelled; + } + Managers::CookedCpuClosureCache::Result assets = {}; + if (!reconstruction.Assets.Poll(&assets)) + return CookedRuntimeReconstructionError::Assets; + if (assets.StateValue == Managers::CookedCpuClosureCache::State::Failed) + return CookedRuntimeReconstructionError::Assets; + if (assets.StateValue == Managers::CookedCpuClosureCache::State::Ready) + { + std::unique_ptr candidate = {}; + if (!assets.Assets || !Rendering::Scenes::RuntimeScene::PrepareFromCooked(std::move(reconstruction.Instance), *assets.Assets, &candidate, scene_diagnostics)) + return CookedRuntimeReconstructionError::Binding; + if (is_cancelled()) + return CookedRuntimeReconstructionError::Cancelled; + *output = std::move(candidate); + *output_assets = std::move(reconstruction.Assets); + return CookedRuntimeReconstructionError::None; + } + // This operation owns the cache worker. No synchronous fallback or + // unbounded wait is allowed when another worker has the pump. + if (!cache.PumpOne(cancelled)) + return is_cancelled() ? CookedRuntimeReconstructionError::Cancelled : CookedRuntimeReconstructionError::Assets; + } + } +} // namespace ZEngine::Applications diff --git a/ZEngine/ZEngine/Applications/CookedRuntimeReconstruction.h b/ZEngine/ZEngine/Applications/CookedRuntimeReconstruction.h new file mode 100644 index 000000000..30a1ff606 --- /dev/null +++ b/ZEngine/ZEngine/Applications/CookedRuntimeReconstruction.h @@ -0,0 +1,41 @@ +#pragma once +#include +#include +#include + +namespace ZEngine::Applications +{ + /// @brief Failure stage for reconstructing a runtime scene from cooked data. + enum class CookedRuntimeReconstructionError : uint8_t + { + None, + InvalidInput, + Cancelled, + Package, + Reconstruction, + Assets, + Binding + }; + + /// @brief Reconstruct and prepare a runtime scene from a cooked package. + /// @details This exclusive worker operation consumes the root and scene slot + /// as it progresses. Success transfers a candidate and closure lease; failure + /// preserves both outputs. The allocator, schemas, cache and VFS provider + /// must outlive the work and returned lease. + [[nodiscard]] CookedRuntimeReconstructionError ReconstructCookedRuntimeScene( + Core::VFS::VFSReadRootLease&& root, + const uuids::uuid& scene, + ECS::SceneStorageLease&& storage, + const ECS::SceneComponentSchemaRegistry& schemas, + const Cook::CookLimits& limits, + Cook::ICookAllocator& allocator, + Core::Memory::IAllocationRoute& route, + size_t max_scene_bytes, + size_t document_working_bytes, + Managers::CookedCpuClosureCache& cache, + std::unique_ptr* output, + Managers::CookedCpuClosureCache::Lease* output_assets, + Cook::CookDiagnostics* cook_diagnostics, + ECS::SceneDiagnostics* scene_diagnostics, + const std::atomic* cancelled = nullptr); +} // namespace ZEngine::Applications diff --git a/ZEngine/ZEngine/Applications/GameApplication.h b/ZEngine/ZEngine/Applications/GameApplication.h index 8099eadf3..c33850238 100644 --- a/ZEngine/ZEngine/Applications/GameApplication.h +++ b/ZEngine/ZEngine/Applications/GameApplication.h @@ -60,22 +60,28 @@ namespace ZEngine::Applications virtual void OverrideWindowConfiguration() = 0; - virtual void OnInitializing() = 0; - virtual void OnInitialized() = 0; + /// Request denial of project-source services as soon as they are created. + [[nodiscard]] virtual bool WantsSourceIndependenceAudit() const + { + return false; + } + + virtual void OnInitializing() = 0; + virtual void OnInitialized() = 0; - virtual void OnEvent(Core::CoreEvent&) = 0; - virtual void OnUpdate(float dt) = 0; + virtual void OnEvent(Core::CoreEvent&) = 0; + virtual void OnUpdate(float dt) = 0; - virtual void OnPreRender() = 0; - virtual void OnPostRender() = 0; - virtual void OnRenderUI() = 0; + virtual void OnPreRender() = 0; + virtual void OnPostRender() = 0; + virtual void OnRenderUI() = 0; - virtual void OnClosing() = 0; - virtual void OnClosed() = 0; + virtual void OnClosing() = 0; + virtual void OnClosed() = 0; /// Main-thread user close intent, not teardown. Return false to defer or /// cancel (for example, while awaiting unsaved-document consent). - virtual bool OnCloseRequested() + virtual bool OnCloseRequested() { return true; } diff --git a/ZEngine/ZEngine/Applications/RuntimeSceneHost.cpp b/ZEngine/ZEngine/Applications/RuntimeSceneHost.cpp new file mode 100644 index 000000000..05e4196d1 --- /dev/null +++ b/ZEngine/ZEngine/Applications/RuntimeSceneHost.cpp @@ -0,0 +1,94 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace ZEngine::Applications +{ + RuntimeSceneHost::~RuntimeSceneHost() + { + ZENGINE_VALIDATE_ASSERT(!m_active, "Runtime scene host must retire its scene after readers join") + } + + bool RuntimeSceneHost::Publish(EngineContext& engine, GameApplication& app, std::unique_ptr& candidate, ECS::SceneRequestId request, uint64_t generation, const uuids::uuid& camera, RenderFrameState frame, ECS::SceneDiagnostics* diagnostics) + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Main.RuntimePublication"); + if (!candidate || !request || !generation || generation == std::numeric_limits::max() || !engine.ScenePublication.Idle() || engine.ScenePublication.Generation() != generation || engine.CloseRequested.value.load(std::memory_order_acquire) || !app.RenderPipeline || !engine.RenderResourceManager || !frame.RenderTargetW || !frame.RenderTargetH) + return false; + if (m_active && (engine.Scene != &m_active->Instance().World() || engine.ActorManager != &m_active->Instance().Actors() || engine.WorldCommands != &m_active->Instance().Commands() || engine.WorldTick != &m_active->Instance().Tick() || app.CurrentScene != &m_active->Render())) + return false; + Rendering::RenderResourceManager::SceneResourcePreparationResult result = {}; + if (!candidate->Resources().Poll(&result) || result.Id.Request != request || result.ExpectedGeneration != generation || result.State != Rendering::RenderResourceManager::SceneResourcePreparationState::ResourcesRetained) + return false; + if (!ECS::CaptureSceneRuntimeCamera(candidate->Instance().World(), camera, frame.RenderTargetW, frame.RenderTargetH, &frame.Camera, diagnostics)) + return false; + frame.SceneGeneration = generation + 1; + frame.SceneResources = candidate->Resources().InitialResources(); + frame.ShowGrid = false; + frame.Gizmo.Reset(); + frame.RenderOverlay = app.EnableRenderOverlay; + frame.FramebufferW = app.CurrentWindow ? app.CurrentWindow->GetWidth() : 0; + frame.FramebufferH = app.CurrentWindow ? app.CurrentWindow->GetHeight() : 0; + AppRenderPipeline::PreparedFrameState prepared = {}; + if (!app.RenderPipeline->PrepareFrameState(frame, &candidate->Render(), &prepared)) + return false; + ECS::ScenePublication::Transition transition = {}; + std::unique_ptr retired = {}; + // Preparation above can fail. Once the generation and request guards + // are held, switch all bindings and publish the reserved frame together. + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Main.RuntimeGuardedCommit"); + if (!engine.ScenePublication.TryBegin(generation, &transition) || !engine.SceneLoads.BeginCommit(request, generation)) + return false; + if (!m_active) + { + m_original_world = engine.Scene; + m_original_actors = engine.ActorManager; + m_original_commands = engine.WorldCommands; + m_original_tick = engine.WorldTick; + m_original_render = app.CurrentScene; + } + retired = std::move(m_active); + m_active = std::move(candidate); + engine.Scene = &m_active->Instance().World(); + engine.ActorManager = &m_active->Instance().Actors(); + engine.WorldCommands = &m_active->Instance().Commands(); + engine.WorldTick = &m_active->Instance().Tick(); + app.CurrentScene = &m_active->Render(); + transition.Publish(); + prepared.Publish(); + const bool committed = engine.SceneLoads.Commit(request, engine.ScenePublication.Generation()); + ZENGINE_VALIDATE_ASSERT(committed, "Runtime scene publication lost its admitted result") + (void) committed; + } + return true; + } + + bool RuntimeSceneHost::ShutdownAfterReadersJoined(EngineContext& engine, GameApplication& app) noexcept + { + if (!m_active) + return true; + if (engine.Scene != &m_active->Instance().World() || engine.ActorManager != &m_active->Instance().Actors() || engine.WorldCommands != &m_active->Instance().Commands() || engine.WorldTick != &m_active->Instance().Tick() || app.CurrentScene != &m_active->Render()) + return false; + engine.Scene = m_original_world; + engine.ActorManager = m_original_actors; + engine.WorldCommands = m_original_commands; + engine.WorldTick = m_original_tick; + app.CurrentScene = m_original_render; + m_active.reset(); + m_original_world = nullptr; + m_original_actors = nullptr; + m_original_commands = nullptr; + m_original_tick = nullptr; + m_original_render = nullptr; + return true; + } + Rendering::Scenes::RuntimeScene* RuntimeSceneHost::Active() const noexcept + { + return m_active.get(); + } + +} // namespace ZEngine::Applications diff --git a/ZEngine/ZEngine/Applications/RuntimeSceneHost.h b/ZEngine/ZEngine/Applications/RuntimeSceneHost.h new file mode 100644 index 000000000..1493acf7c --- /dev/null +++ b/ZEngine/ZEngine/Applications/RuntimeSceneHost.h @@ -0,0 +1,43 @@ +#pragma once +#include + +namespace ZEngine +{ + struct EngineContext; +} +namespace ZEngine::Applications +{ + struct GameApplication; + struct RenderFrameState; + + /// @brief Owns publication of renderer-only runtime scenes on the main thread. + /// @details The caller must invoke ShutdownAfterReadersJoined before engine + /// services are destroyed. The currently available scene preparation path + /// consumes cooked assets, but publication does not inspect their format. + class RuntimeSceneHost final + { + public: + RuntimeSceneHost() = default; + ~RuntimeSceneHost(); + RuntimeSceneHost(const RuntimeSceneHost&) = delete; + RuntimeSceneHost& operator=(const RuntimeSceneHost&) = delete; + + /// @brief Publish a prepared scene for the expected generation. + /// @details Validation and frame reservation precede the guarded no-fail + /// switch. Failure preserves the candidate and active bindings. + [[nodiscard]] bool Publish(EngineContext&, GameApplication&, std::unique_ptr&, ECS::SceneRequestId, uint64_t expected_generation, const uuids::uuid& camera, RenderFrameState, ECS::SceneDiagnostics*); + /// @brief Restore the previous bindings and retire the active scene. + /// @details Call after render and pool workers join, before RRM shutdown. + [[nodiscard]] bool ShutdownAfterReadersJoined(EngineContext&, GameApplication&) noexcept; + /// @brief Return the active scene, or nullptr before publication. + [[nodiscard]] Rendering::Scenes::RuntimeScene* Active() const noexcept; + + private: + std::unique_ptr m_active = {}; + ECS::Scene* m_original_world = nullptr; + ECS::ActorManager* m_original_actors = nullptr; + ECS::WorldCommands* m_original_commands = nullptr; + ECS::WorldTick* m_original_tick = nullptr; + Rendering::Scenes::RenderScene* m_original_render = nullptr; + }; +} // namespace ZEngine::Applications diff --git a/ZEngine/ZEngine/Applications/StandaloneRuntimeApplication.cpp b/ZEngine/ZEngine/Applications/StandaloneRuntimeApplication.cpp new file mode 100644 index 000000000..ffe7c8167 --- /dev/null +++ b/ZEngine/ZEngine/Applications/StandaloneRuntimeApplication.cpp @@ -0,0 +1,1069 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ZEngine::Applications +{ + namespace + { + constexpr size_t CpuBytes = ZMega(1024ULL); + constexpr size_t GpuBytes = ZMega(256ULL); + constexpr size_t MetadataBytes = ZMega(16ULL); + constexpr size_t ObjectBytes = ZMega(512ULL); + uint64_t TimestampNs() + { + return static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); + } + } // namespace + + void StandaloneRuntimeApplication::Configure(std::string directory, uuids::uuid scene, uuids::uuid camera, bool exit_after_frame, uint32_t expected_draws, uint32_t reloads, uuids::uuid alternate, bool retry_failure_probe, bool cancel_upload_probe, bool cancel_cpu_probe, bool supersede_cpu_probe, bool supersede_gpu_probe, bool supersede_waiting_probe) + { + m_directory = std::move(directory); + m_scene = scene; + m_initial_scene = scene; + m_alternate_scene = alternate; + m_reloads_remaining = reloads; + m_retry_failure_probe = retry_failure_probe; + m_cancel_upload_probe = cancel_upload_probe; + m_cancel_cpu_probe = cancel_cpu_probe; + m_supersede_cpu_probe = supersede_cpu_probe; + m_supersede_gpu_probe = supersede_gpu_probe; + m_supersede_waiting_probe = supersede_waiting_probe; + m_camera = camera; + m_exit_after_frame = exit_after_frame; + m_expected_draws = expected_draws; + EnableRenderOverlay = true; + } + + bool StandaloneRuntimeApplication::BeginCapture(void* context, uint32_t width, uint32_t height, uint64_t generation) + { + auto& app = *static_cast(context); + if (generation != app.m_capture_generation || width != app.m_capture_width || height != app.m_capture_height) + { + uint32_t requested = 1; + app.m_capture_state.compare_exchange_strong(requested, 4, std::memory_order_acq_rel); + return false; + } + uint32_t requested = 1; + return app.m_capture_state.compare_exchange_strong(requested, 2, std::memory_order_acq_rel); + } + + void StandaloneRuntimeApplication::CompleteCapture(const void* data, size_t bytes, void* context) + { + auto& app = *static_cast(context); + if (!data || bytes != app.m_capture_bytes || !app.m_capture_pixels) + { + app.m_capture_state.store(4, std::memory_order_release); + return; + } + std::memcpy(app.m_capture_pixels.get(), data, bytes); + app.m_capture_state.store(3, std::memory_order_release); + } + + void StandaloneRuntimeApplication::CancelCapture(void* context) + { + static_cast(context)->m_capture_state.store(4, std::memory_order_release); + } + + void StandaloneRuntimeApplication::OnInitializing() + { + Memory->CreateBudgetedArena(Memory->Budget.EditorContext, &m_arena); + } + + void StandaloneRuntimeApplication::OverrideWindowConfiguration() + { + WindowCfg.Title.init(&m_arena, "Standalone runtime"); + WindowCfg.EnableVsync = true; + if (m_lifecycle_stress || m_fixed_test_window) + { + WindowCfg.Width = 1280; + WindowCfg.Height = 720; + WindowCfg.Resizable = false; + WindowCfg.Maximize = false; + } + } + + void StandaloneRuntimeApplication::Fail(const char* message, ECS::SceneLoadStage stage) + { + if (!m_failed) + ZENGINE_CORE_ERROR("[CookedRuntime] {}", message) + RestoreStagingLimit(m_gpu.Status() == Rendering::Scenes::CookedSceneResourceRequest::State::Failed); + auto* context = Engine::GetContext(); + if (context && context->Device && context->Device->IsDeviceLost.load(std::memory_order_acquire)) + { + m_failed = true; + ZENGINE_CORE_ERROR("[CookedRuntime] Device lost; entering fatal shutdown") + Engine::RequestClose(); + return; + } + if (m_host.Active() && m_request) + { + auto* engine = Engine::GetContext(); + m_gpu.Cancel(); + const auto status = m_gpu.Tick(); + ZENGINE_VALIDATE_ASSERT(status != Rendering::Scenes::CookedSceneResourceRequest::State::Cancelling && status != Rendering::Scenes::CookedSceneResourceRequest::State::GpuPending, "Runtime failed request still has GPU work") + m_candidate.reset(); + m_gpu.ReleaseTerminalAssets(); + const bool reset = m_gpu.Reset(); + ZENGINE_VALIDATE_ASSERT(reset, "Runtime failed request did not reset") + m_assets.Reset(); + m_root.Reset(); + m_storage.Reset(); + if (engine->SceneLoads.Pending(m_request)) + engine->SceneLoads.Fail(m_request, stage, ECS::SceneLoadError::Resources); + const bool cleaned = engine->SceneLoads.FinishCleanup(m_request); + const bool acknowledged = m_request_retain_result || engine->SceneLoads.Acknowledge(m_request); + ZENGINE_VALIDATE_ASSERT(cleaned && acknowledged, "Runtime failed request cleanup failed") + if (m_probe_old == m_request) + { + ECS::SceneLoadResult outcome = {}; + if (!PollSceneRequest(m_probe_old, &outcome) || outcome.Outcome != ECS::SceneLoadOutcome::Failed || outcome.Stage != stage || outcome.CleanupPending || outcome.Error != ECS::SceneLoadError::Resources || outcome.ExpectedGeneration != engine->ScenePublication.Generation() || outcome.CommittedGeneration != 0 || CancelSceneRequest(m_probe_old) != ECS::SceneCancelResult::AlreadyTerminal || !AcknowledgeSceneRequest(m_probe_old) || PollSceneRequest(m_probe_old, &outcome)) + { + m_failed = true; + ZENGINE_CORE_ERROR("[CookedRuntime] Failed replacement outcome was not retained correctly") + Engine::RequestClose(); + return; + } + m_probe_old = {}; + ZENGINE_CORE_INFO("[CookedRuntime] Failed replacement outcome retained and acknowledged") + } + m_request = {}; + m_request_retain_result = false; + ZENGINE_CORE_INFO("[CookedRuntime] Replacement failed; active scene generation {} retained", engine->ScenePublication.Generation()) + if (m_retry_failure_probe) + { + Rendering::Scenes::SceneSubmissionProgress::Snapshot submitted = {}; + m_failure_submission_baseline = RenderPipeline->SceneSubmissions().TryRead(&submitted) && submitted.Generation == engine->ScenePublication.Generation() ? submitted.Count : 0; + m_waiting_failure_frame = true; + } + return; + } + m_failed = true; + Engine::RequestClose(); + } + + void StandaloneRuntimeApplication::RestoreStagingLimit(bool request_failed) noexcept + { + if (!m_staging_limit_active) + return; + auto* engine = Engine::GetContext(); + auto& allocator = engine->Device->GpuMem; + const auto stats = allocator.GetAllocationStatistics(); + const auto domain = static_cast(Core::Memory::GpuMemoryDomain::HostStaging); + m_staging_refusal_observed = request_failed && m_staging_gpu_started && stats.DomainBudgetRefusals[domain] > m_staging_refusal_baseline; + const bool restored = allocator.SetAllocationLimits(m_saved_gpu_limits); + ZENGINE_VALIDATE_ASSERT(restored, "Runtime test staging limit could not be restored") + (void) restored; + m_staging_limit_active = false; + if (m_staging_refusal_observed) + ZENGINE_CORE_INFO("[CookedRuntime] Replacement staging allocation was refused") + } + + void StandaloneRuntimeApplication::OnInitialized() + { + auto* engine = Engine::GetContext(); + if (m_source_audit && (!engine || !engine->VFS || !engine->ImportCoordinator || !Managers::AssetManager::Instance() || !Managers::AssetManager::Instance()->Registry)) + return Fail("cannot activate source-independence audit"); + if (!engine || m_directory.empty() || m_scene.is_nil() || !engine->SceneSchemas || !engine->SceneDocumentWorkingBytes || engine->SceneDocumentWorkingBytes > std::numeric_limits::max() - ECS::SceneStorageLease::MinimumBytes || !engine->SceneSchemas->FreezeForWorker(&m_scene_diagnostics) || m_cpu_allocator.Initialize(CpuBytes).Failed() || m_gpu_allocator.Initialize(GpuBytes).Failed() || !m_metadata.TryInit(&m_arena, MetadataBytes)) + return Fail("cannot initialize bounded runtime services"); + const Managers::CookedCpuAssetLimits limits = {.MaxAssets = 8192, .MaxObjectBytes = ObjectBytes, .MaxMeshWorkspaceBytes = ObjectBytes}; + if (!m_cache.Initialize(m_cpu_route, *engine->SceneSchemas, limits, 2, 32, 8192)) + return Fail("cannot initialize cooked closure cache"); + m_cache_initialized = true; + m_backend.Initialize(m_directory.c_str(), Core::VFS::VFSBackendCaps::Read | Core::VFS::VFSBackendCaps::List, &m_arena); + m_vfs.Initialize(&m_arena); + m_vfs_initialized = true; + if (m_vfs.Mount(&m_backend, Core::VFS::VFSPath::Root(), 0).Failed() || m_vfs.AcquireReadRoot(Core::VFS::VFSPath::Root(), &m_root).Failed() || engine->AdmitSceneStorage(&m_storage, engine->SceneDocumentWorkingBytes + ECS::SceneStorageLease::MinimumBytes) != ECS::SceneStorageAdmission::Acquired) + return Fail("cannot admit cooked package or scene storage"); + m_generation = engine->ScenePublication.Generation(); + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Main.BeforeInitialAdmission"); + } + m_admission_ns = TimestampNs(); + if (!engine->SceneLoads.Admit(m_generation, &m_request) || !engine->SceneLoads.TryStart(m_request)) + return Fail("cannot admit cooked scene request"); + m_width = CurrentWindow ? CurrentWindow->GetWidth() : 0; + m_height = CurrentWindow ? CurrentWindow->GetHeight() : 0; + m_initialized = true; + } + + void StandaloneRuntimeApplication::RunWorker(void* context) + { + auto& self = *static_cast(context); + auto* engine = Engine::GetContext(); + if (self.m_cpu_pause_armed) + { + self.m_cpu_boundary.store(true, std::memory_order_release); + while (!self.m_cancelled.value.load(std::memory_order_acquire)) + std::this_thread::yield(); + } + while (self.m_cache.PumpOne(&self.m_cancelled.value)) + { + } + const Cook::CookLimits limits = {.MaxWorkingBytes = CpuBytes}; + self.m_worker_result = ReconstructCookedRuntimeScene(std::move(self.m_root), self.m_scene, std::move(self.m_storage), *engine->SceneSchemas, limits, self.m_cpu_allocator, self.m_cpu_route, ObjectBytes, engine->SceneDocumentWorkingBytes, self.m_cache, &self.m_candidate, &self.m_assets, &self.m_cook_diagnostics, &self.m_scene_diagnostics, &self.m_cancelled.value); + ZENGINE_CORE_INFO("[CookedRuntime] Worker finished with status {}", static_cast(self.m_worker_result)) + self.m_worker_done.value.store(true, std::memory_order_release); + } + + void StandaloneRuntimeApplication::DrainCacheWorker(void* context) + { + auto& self = *static_cast(context); + while (self.m_cache.PumpOne()) + { + } + self.m_cache_drain_running.value.store(false, std::memory_order_release); + } + + bool StandaloneRuntimeApplication::RequestScene(const uuids::uuid& scene, ECS::SceneRequestId* output) + { + auto* engine = Engine::GetContext(); + if (!m_initialized || m_failed || !engine || !m_host.Active() || scene.is_nil() || engine->CloseRequested.value.load(std::memory_order_acquire)) + return false; + const uint64_t generation = engine->ScenePublication.Generation(); + const uint64_t admission_ns = TimestampNs(); + const bool preparing_was_pending = m_request && engine->SceneLoads.Pending(m_request); + ECS::SceneRequestId request = {}; + if (!engine->SceneLoads.Admit(generation, &request)) + return false; + if (m_waiting_request) + { + ECS::SceneLoadResult previous = {}; + const bool superseded = engine->SceneLoads.Poll(m_waiting_request, &previous) && previous.Outcome == ECS::SceneLoadOutcome::Superseded && !previous.CleanupPending; + const bool acknowledged = m_waiting_retain_result || engine->SceneLoads.Acknowledge(m_waiting_request); + ZENGINE_VALIDATE_ASSERT(superseded && acknowledged, "Superseded waiting request lost its terminal outcome") + m_waiting_superseded_observed = superseded && acknowledged; + } + m_waiting_request = request; + m_waiting_retain_result = output != nullptr; + m_waiting_scene = scene; + m_waiting_generation = generation; + m_waiting_admission_ns = admission_ns; + if (m_request) + { + if (preparing_was_pending) + m_request_terminal = ECS::SceneLoadOutcome::Superseded; + m_cancelled.value.store(true, std::memory_order_release); + m_gpu.Cancel(); + } + if (output) + *output = request; + return true; + } + + void StandaloneRuntimeApplication::StartWaiting() + { + auto* engine = Engine::GetContext(); + if (!engine || !m_waiting_request || m_request || m_candidate) + return; + if (!engine->SceneLoads.Pending(m_waiting_request) || m_waiting_generation != engine->ScenePublication.Generation()) + { + if (engine->SceneLoads.Pending(m_waiting_request)) + (void) engine->SceneLoads.Cancel(m_waiting_request); + if (!m_waiting_retain_result) + (void) engine->SceneLoads.Acknowledge(m_waiting_request); + m_waiting_request = {}; + m_waiting_retain_result = false; + return; + } + ECS::SceneStorageLease storage = {}; + Core::VFS::VFSReadRootLease root = {}; + const auto admission = engine->AdmitSceneStorage(&storage, engine->SceneDocumentWorkingBytes + ECS::SceneStorageLease::MinimumBytes); + if (admission == ECS::SceneStorageAdmission::Busy) + return; + if (admission != ECS::SceneStorageAdmission::Acquired || m_vfs.AcquireReadRoot(Core::VFS::VFSPath::Root(), &root).Failed()) + { + (void) engine->SceneLoads.Fail(m_waiting_request, ECS::SceneLoadStage::Admission, ECS::SceneLoadError::Resources); + if (!m_waiting_retain_result) + (void) engine->SceneLoads.Acknowledge(m_waiting_request); + m_waiting_request = {}; + m_waiting_retain_result = false; + ZENGINE_CORE_ERROR("[CookedRuntime] Waiting replacement admission failed; active scene retained") + return; + } + if (!engine->SceneLoads.TryStart(m_waiting_request)) + return; + m_scene = m_waiting_scene; + m_generation = m_waiting_generation; + m_admission_ns = m_waiting_admission_ns; + m_request = m_waiting_request; + m_request_retain_result = m_waiting_retain_result; + m_request_terminal = ECS::SceneLoadOutcome::Pending; + m_waiting_request = {}; + m_waiting_retain_result = false; + m_storage = std::move(storage); + m_root = std::move(root); + m_cook_diagnostics.Clear(); + m_scene_diagnostics.Clear(); + m_worker_result = CookedRuntimeReconstructionError::InvalidInput; + m_worker_done.value.store(false, std::memory_order_relaxed); + m_cpu_boundary.store(false, std::memory_order_relaxed); + m_cancelled.value.store(false, std::memory_order_release); + m_cpu_pause_armed = m_cancel_cpu_probe || m_supersede_cpu_probe || m_shutdown_probe_cpu; + m_cpu_supersede_armed = m_supersede_cpu_probe; + m_cancel_cpu_probe = false; + m_supersede_cpu_probe = false; + m_dispatched = false; + m_gpu_started = false; + m_last_gpu_status = 255; + m_publication_wait_logged = false; + } + + ECS::SceneCancelResult StandaloneRuntimeApplication::CancelSceneRequest() noexcept + { + auto* engine = Engine::GetContext(); + if (!engine || !m_host.Active()) + return ECS::SceneCancelResult::NotFound; + return CancelSceneRequest(m_waiting_request ? m_waiting_request : m_request); + } + + bool StandaloneRuntimeApplication::PollSceneRequest(ECS::SceneRequestId request, ECS::SceneLoadResult* output) const noexcept + { + const auto* engine = Engine::GetContext(); + return engine && engine->SceneLoads.Poll(request, output); + } + + bool StandaloneRuntimeApplication::AcknowledgeSceneRequest(ECS::SceneRequestId request) noexcept + { + auto* engine = Engine::GetContext(); + return engine && engine->SceneLoads.Acknowledge(request); + } + + ECS::SceneCancelResult StandaloneRuntimeApplication::CancelSceneRequest(ECS::SceneRequestId request) noexcept + { + auto* engine = Engine::GetContext(); + if (!engine || !request || !m_host.Active()) + return ECS::SceneCancelResult::NotFound; + const uint64_t cancellation_ns = m_lifecycle_stress && request == m_request ? TimestampNs() : 0; + const auto result = engine->SceneLoads.Cancel(request); + if (result != ECS::SceneCancelResult::Cancelled) + return result; + if (request == m_waiting_request) + { + if (!m_waiting_retain_result) + (void) engine->SceneLoads.Acknowledge(request); + m_waiting_request = {}; + m_waiting_retain_result = false; + } + else if (request == m_request) + { + if (m_lifecycle_stress) + m_stress_cancel_requested_ns = cancellation_ns; + m_request_terminal = ECS::SceneLoadOutcome::Cancelled; + m_cancelled.value.store(true, std::memory_order_release); + m_gpu.Cancel(); + } + return result; + } + + void StandaloneRuntimeApplication::OnUpdate(float) + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Main.RuntimeUpdate"); + if (!m_initialized || m_failed) + return; + auto* engine = Engine::GetContext(); + if (engine->Device && engine->Device->IsDeviceLost.load(std::memory_order_acquire)) + return Fail("device lost during runtime scene work"); + if (m_observe_gpu_completion && engine->Device && engine->Device->SwapchainPtr && engine->Device->SwapchainPtr->RenderTimeline) + { + Rendering::Scenes::SceneSubmissionProgress::Snapshot submitted = {}; + if (RenderPipeline->SceneSubmissions().TryRead(&submitted) && submitted.Generation == engine->ScenePublication.Generation() && submitted.Generation != m_completed_generation && submitted.Count) + { + if (!submitted.FirstTimelineValue) + return Fail("scene submission did not provide its render timeline"); + uint64_t completed = 0; + const VkResult status = vkGetSemaphoreCounterValue(engine->Device->LogicalDevice, engine->Device->SwapchainPtr->RenderTimeline->GetHandle(), &completed); + if (status != VK_SUCCESS) + { + if (status == VK_ERROR_DEVICE_LOST) + (void) engine->Device->CheckDeviceLost(status, "runtime completion observation"); + return Fail("cannot observe scene render completion"); + } + if (completed >= submitted.FirstTimelineValue) + { + m_completed_generation = submitted.Generation; + m_completion_observed_ns = TimestampNs(); + } + } + } + if (m_request && m_cpu_pause_armed && !m_cpu_cancellation_observed && m_cpu_boundary.load(std::memory_order_acquire)) + { + if (m_shutdown_probe_cpu && m_host.Active()) + { + m_shutdown_probe_started = true; + m_shutdown_probe_generation = engine->ScenePublication.Generation(); + m_frame_recorded = true; + ZENGINE_CORE_INFO("[CookedRuntime] Shutdown requested with CPU worker pending") + Engine::RequestClose(); + return; + } + if (m_cpu_supersede_armed) + { + const auto old = m_request; + ECS::SceneLoadResult outcome = {}; + if (!RequestScene(m_initial_scene) || !engine->SceneLoads.Poll(old, &outcome) || outcome.Outcome != ECS::SceneLoadOutcome::Superseded || !outcome.CleanupPending) + return Fail("cannot supersede runtime CPU worker"); + if (m_probe_old == old) + { + if (!AcknowledgeSceneRequest(old) || PollSceneRequest(old, &outcome)) + return Fail("cannot acknowledge superseded worker outcome"); + m_probe_old = {}; + ZENGINE_CORE_INFO("[CookedRuntime] Superseded worker outcome acknowledged before cleanup") + } + m_cpu_supersede_armed = false; + m_cpu_cancellation_observed = true; + ZENGINE_CORE_INFO("[CookedRuntime] Superseded CPU worker with cleanup pending") + return; + } + if (CancelSceneRequest() != ECS::SceneCancelResult::Cancelled) + return Fail("cannot cancel runtime CPU worker"); + if (m_probe_old == m_request) + { + ECS::SceneLoadResult outcome = {}; + if (!PollSceneRequest(m_probe_old, &outcome) || outcome.Outcome != ECS::SceneLoadOutcome::Cancelled || !outcome.CleanupPending || !AcknowledgeSceneRequest(m_probe_old) || PollSceneRequest(m_probe_old, &outcome)) + return Fail("cannot acknowledge cancelled worker outcome"); + m_probe_old = {}; + ZENGINE_CORE_INFO("[CookedRuntime] Cancelled worker outcome acknowledged before cleanup") + } + m_cpu_cancellation_observed = true; + ZENGINE_CORE_INFO("[CookedRuntime] Cancelled replacement while CPU worker owned its inputs") + } + if (m_request && !engine->SceneLoads.Pending(m_request)) + { + const auto outcome = m_request_terminal; + if (outcome != ECS::SceneLoadOutcome::Cancelled && outcome != ECS::SceneLoadOutcome::Superseded) + return Fail("lost runtime request outcome"); + m_cancelled.value.store(true, std::memory_order_release); + if (m_dispatched && !m_worker_done.value.load(std::memory_order_acquire)) + return; + m_gpu.Cancel(); + const auto state = m_gpu.Tick(); + if (state == Rendering::Scenes::CookedSceneResourceRequest::State::Cancelling || state == Rendering::Scenes::CookedSceneResourceRequest::State::GpuPending) + return; + m_candidate.reset(); + m_gpu.ReleaseTerminalAssets(); + const bool reset = m_gpu.Reset(); + m_assets.Reset(); + m_root.Reset(); + m_storage.Reset(); + const bool cleaned = engine->SceneLoads.FinishCleanup(m_request); + const bool acknowledged = m_request_retain_result || engine->SceneLoads.Acknowledge(m_request); + ZENGINE_VALIDATE_ASSERT(reset && cleaned && acknowledged, "Runtime cancelled request cleanup failed") + m_request = {}; + m_request_retain_result = false; + if (outcome == ECS::SceneLoadOutcome::Superseded) + { + ZENGINE_CORE_INFO("[CookedRuntime] Superseded replacement drained; active scene retained") + } + else + { + ZENGINE_CORE_INFO("[CookedRuntime] Cancelled replacement drained; active scene generation {} retained", engine->ScenePublication.Generation()) + } + if (m_lifecycle_stress && outcome == ECS::SceneLoadOutcome::Cancelled) + { + if (engine->ScenePublication.Generation() != m_stress_cancel_generation) + return Fail("stress cancellation changed the active generation"); + Rendering::Scenes::SceneSubmissionProgress::Snapshot submitted = {}; + if (!RenderPipeline->SceneSubmissions().TryRead(&submitted) || submitted.Generation != m_stress_cancel_generation) + return Fail("stress cancellation lost the active scene submission"); + m_stress_submission_baseline = submitted.Count; + m_stress_waiting_frame = true; + ++m_stress_cancelled; + const auto cleanup_ns = TimestampNs(); + if (!m_stress_cancel_requested_ns || cleanup_ns < m_stress_cancel_requested_ns) + return Fail("invalid stress cancellation timing"); + ZENGINE_CORE_INFO("[CookedCancelTiming] cancelled={} boundary={} request_ns={} cleanup_ns={} elapsed_ns={}", m_stress_cancelled, (m_stress_cancelled % 2) ? "cpu" : "gpu", m_stress_cancel_requested_ns, cleanup_ns, cleanup_ns - m_stress_cancel_requested_ns) + m_stress_cancel_requested_ns = 0; + } + if (m_retry_failure_probe && !m_waiting_request) + { + Rendering::Scenes::SceneSubmissionProgress::Snapshot submitted = {}; + m_failure_submission_baseline = RenderPipeline->SceneSubmissions().TryRead(&submitted) && submitted.Generation == engine->ScenePublication.Generation() ? submitted.Count : 0; + m_waiting_failure_frame = true; + } + return; + } + if (!m_request && m_waiting_request) + { + StartWaiting(); + return; + } + if (m_request && !m_dispatched) + { + if (m_cache_drain_running.value.load(std::memory_order_acquire)) + return; + auto* pool = Helpers::ThreadPoolHelper::Pool.get(); + if (pool && pool->Submit(this, &RunWorker)) + m_dispatched = true; + return; + } + if (m_request && !m_worker_done.value.load(std::memory_order_acquire)) + return; + if (m_request && m_worker_result != CookedRuntimeReconstructionError::None) + { + if (m_worker_result == CookedRuntimeReconstructionError::Assets) + ZENGINE_CORE_ERROR("[CookedRuntime] Asset closure reconstruction failed") + for (uint32_t i = 0; i < m_cook_diagnostics.Count; ++i) + ZENGINE_CORE_ERROR("[CookedRuntime] {}", m_cook_diagnostics.Entries[i].Message) + for (uint32_t i = 0; i < m_scene_diagnostics.Count; ++i) + ZENGINE_CORE_ERROR("[CookedRuntime] {}", m_scene_diagnostics.Entries[i].Message) + return Fail("cooked reconstruction failed"); + } + if (m_request) + { + if (!m_width || !m_height) + return; + if (!m_gpu_started) + { + RenderFrameState frame = {}; + frame.RenderTargetW = m_width; + frame.RenderTargetH = m_height; + frame.ResizeSequence = m_resize_sequence; + ECS::Systems::SyncHierarchy(m_candidate->Instance().World()); + if (!ECS::CaptureSceneRuntimeCamera(m_candidate->Instance().World(), m_camera, m_width, m_height, &frame.Camera, &m_scene_diagnostics)) + { + for (uint32_t i = 0; i < m_scene_diagnostics.Count; ++i) + ZENGINE_CORE_ERROR("[CookedRuntime] {}", m_scene_diagnostics.Entries[i].Message) + return Fail("runtime camera is invalid"); + } + if (!m_host.Active()) + { + m_bootstrap.Instances.m_allocator = &m_arena; + m_bootstrap.Sky.Mode = Rendering::Scenes::SkyMode::SkySphere; + m_bootstrap_camera = frame.Camera; + CurrentScene = &m_bootstrap; + } + const Rendering::Scenes::SceneInitialView view = {frame.Camera, m_width, m_height, m_resize_sequence, false, true}; + ZENGINE_CORE_INFO("[CookedRuntime] Candidate sky mode {}", static_cast(m_candidate->Render().Sky.Mode)) + if (m_host.Active() && m_fail_staging_probe && !m_staging_limit_active) + { + auto& allocator = engine->Device->GpuMem; + const auto stats = allocator.GetAllocationStatistics(); + const auto domain = static_cast(Core::Memory::GpuMemoryDomain::HostStaging); + auto limited = allocator.GetAllocationLimits(); + m_saved_gpu_limits = limited; + limited.DomainBytes[domain] = stats.DomainAdmittedBytes[domain]; + if (!allocator.SetAllocationLimits(limited)) + return; + m_staging_refusal_baseline = stats.DomainBudgetRefusals[domain]; + m_staging_limit_active = true; + m_fail_staging_probe = false; + } + if (!m_gpu.Begin(m_candidate->Instance(), m_candidate->Render(), m_candidate->Resources(), m_candidate->RequiredMeshes(), std::move(m_assets), m_request, m_generation, view, m_gpu_route, m_metadata, *engine->RenderResourceManager, *engine->Device, RenderPipeline->CookedSceneUploads, Rendering::RenderResourceManager::SceneCookedTextureCapacity, m_host.Active() && (m_cancel_upload_probe || m_supersede_gpu_probe || m_device_loss_probe || m_shutdown_probe_gpu))) + return Fail("cannot begin cooked GPU preparation"); + if (m_host.Active() && m_reject_upload_probe) + { + engine->RenderResourceManager->ArmCookedBatchRejectionProbe(); + m_reject_upload_probe = false; + } + m_gpu_started = true; + // Camera selection and resource admission both scale with the + // candidate. Give each its own update rather than charging both + // to the same main-thread frame at the scene-size limit. + return; + } + const auto state = m_gpu.Tick(); + if (m_shutdown_probe_gpu && m_host.Active() && state == Rendering::Scenes::CookedSceneResourceRequest::State::GpuPending && m_gpu.MeshUploadRecorded()) + { + m_shutdown_probe_started = true; + m_shutdown_probe_generation = engine->ScenePublication.Generation(); + m_frame_recorded = true; + ZENGINE_CORE_INFO("[CookedRuntime] Shutdown requested with GPU upload pending") + Engine::RequestClose(); + return; + } + if (m_staging_limit_active && state == Rendering::Scenes::CookedSceneResourceRequest::State::GpuPending) + m_staging_gpu_started = true; + if (m_device_loss_probe && m_host.Active() && state == Rendering::Scenes::CookedSceneResourceRequest::State::GpuPending && m_gpu.MeshUploadRecorded()) + { + m_device_loss_probe = false; + ZENGINE_CORE_INFO("[CookedRuntime] Injecting device loss after upload recording") + (void) engine->Device->CheckDeviceLost(VK_ERROR_DEVICE_LOST, "injected runtime upload loss"); + return Fail("device lost during runtime scene work"); + } + if (m_supersede_gpu_probe && m_host.Active() && state == Rendering::Scenes::CookedSceneResourceRequest::State::GpuPending && m_gpu.MeshUploadRecorded()) + { + const auto old = m_request; + ECS::SceneLoadResult outcome = {}; + if (!RequestScene(m_initial_scene) || !engine->SceneLoads.Poll(old, &outcome) || outcome.Outcome != ECS::SceneLoadOutcome::Superseded || !outcome.CleanupPending) + return Fail("cannot supersede recorded runtime upload"); + m_supersede_gpu_probe = false; + ZENGINE_CORE_INFO("[CookedRuntime] Superseded recorded upload with cleanup pending") + return; + } + if (m_cancel_upload_probe && m_host.Active() && state == Rendering::Scenes::CookedSceneResourceRequest::State::GpuPending && m_gpu.MeshUploadRecorded()) + { + if (CancelSceneRequest() != ECS::SceneCancelResult::Cancelled) + return Fail("cannot cancel recorded runtime upload"); + if (m_probe_old == m_request) + { + ECS::SceneLoadResult outcome = {}; + if (!PollSceneRequest(m_probe_old, &outcome) || outcome.Outcome != ECS::SceneLoadOutcome::Cancelled || !outcome.CleanupPending || !AcknowledgeSceneRequest(m_probe_old) || PollSceneRequest(m_probe_old, &outcome)) + return Fail("cannot acknowledge cancelled upload outcome"); + m_probe_old = {}; + ZENGINE_CORE_INFO("[CookedRuntime] Cancelled upload outcome acknowledged before cleanup") + } + m_cancel_upload_probe = false; + m_upload_cancellation_observed = true; + ZENGINE_CORE_INFO("[CookedRuntime] Cancelled replacement after mesh upload recording") + return; + } + if (m_last_gpu_status != static_cast(state)) + { + m_last_gpu_status = static_cast(state); + ZENGINE_CORE_INFO("[CookedRuntime] GPU preparation state {}", static_cast(state)) + } + if (state == Rendering::Scenes::CookedSceneResourceRequest::State::Failed || state == Rendering::Scenes::CookedSceneResourceRequest::State::Cancelled) + return Fail("cooked GPU preparation failed"); + if (state != Rendering::Scenes::CookedSceneResourceRequest::State::Ready) + return; + if (m_host.Active() && m_fail_preflight_probe) + { + m_fail_preflight_probe = false; + return Fail("injected publication preflight refusal", ECS::SceneLoadStage::Publication); + } + RenderFrameState frame = {}; + frame.RenderTargetW = m_width; + frame.RenderTargetH = m_height; + frame.ResizeSequence = m_resize_sequence; + if (!m_host.Publish(*engine, *this, m_candidate, m_request, m_generation, m_camera, frame, &m_scene_diagnostics)) + { + if (!m_publication_wait_logged) + ZENGINE_CORE_WARN("[CookedRuntime] Waiting for scene publication preflight") + m_publication_wait_logged = true; + return; + } + if (!m_gpu.TakeReady()) + return Fail("cannot finish cooked GPU preparation"); + const bool cleaned = engine->SceneLoads.FinishCleanup(m_request); + const bool acknowledged = m_request_retain_result || engine->SceneLoads.Acknowledge(m_request); + ZENGINE_VALIDATE_ASSERT(cleaned && acknowledged, "Runtime scene request cleanup failed") + m_request = {}; + m_request_retain_result = false; + m_active_scene = m_scene; + if (m_lifecycle_stress && m_stress_admitted) + ++m_stress_committed; + m_active_admission_ns = m_admission_ns; + ZENGINE_CORE_INFO("[CookedRuntime] Published cooked scene generation {} scene={}", engine->ScenePublication.Generation(), uuids::to_string(m_active_scene)) + if (m_probe_latest) + { + ECS::SceneLoadResult outcome = {}; + if (!PollSceneRequest(m_probe_latest, &outcome) || outcome.Outcome != ECS::SceneLoadOutcome::Committed || outcome.CommittedGeneration != engine->ScenePublication.Generation() || outcome.CleanupPending || CancelSceneRequest(m_probe_latest) != ECS::SceneCancelResult::TooLate || !AcknowledgeSceneRequest(m_probe_latest) || PollSceneRequest(m_probe_latest, &outcome)) + return Fail("runtime committed request outcome was not retained correctly"); + m_probe_latest = {}; + ZENGINE_CORE_INFO("[CookedRuntime] Committed request outcome retained and acknowledged") + } + } + if (!m_request && !m_waiting_request && !m_cache_drain_running.value.load(std::memory_order_acquire)) + { + Managers::CookedCpuClosureCache::Statistics cache = {}; + if (m_cache.TryGetStatistics(&cache) && cache.Retiring) + { + auto* pool = Helpers::ThreadPoolHelper::Pool.get(); + m_cache_drain_running.value.store(true, std::memory_order_release); + if (!pool || !pool->Submit(this, &DrainCacheWorker)) + m_cache_drain_running.value.store(false, std::memory_order_release); + } + } + if (m_exit_after_frame) + { + Rendering::Scenes::SceneSubmissionProgress::Snapshot submitted = {}; + if (m_stress_waiting_frame) + { + if (!RenderPipeline->SceneSubmissions().TryRead(&submitted) || submitted.Generation != m_stress_cancel_generation || submitted.Count <= m_stress_submission_baseline) + return; + m_stress_waiting_frame = false; + ZENGINE_CORE_INFO("[CookedStress] cancelled={} generation={} rendered_after_cleanup=1", m_stress_cancelled, submitted.Generation) + } + if (m_waiting_failure_frame) + { + if (RenderPipeline->SceneSubmissions().TryRead(&submitted) && submitted.Generation == engine->ScenePublication.Generation() && submitted.Count > m_failure_submission_baseline && RequestScene(m_initial_scene)) + { + ZENGINE_CORE_INFO("[CookedRuntime] Active scene submitted after failure; retry admitted") + m_waiting_failure_frame = false; + m_retry_failure_probe = false; + } + return; + } + if (RenderPipeline->SceneSubmissions().TryRead(&submitted) && submitted.Generation == engine->ScenePublication.Generation() && submitted.Count >= 8) + { + if (m_expected_draws && submitted.DrawCount != m_expected_draws) + return Fail("submitted draw count does not match fixture"); + if (m_reported_generation != submitted.Generation) + { + if (m_observe_gpu_completion && m_completed_generation != submitted.Generation) + return; + Managers::CookedCpuClosureCache::Statistics cache = {}; + if (!m_cache.TryGetStatistics(&cache)) + return; + if (cache.Retiring || m_cache_drain_running.value.load(std::memory_order_acquire)) + return; + if (!m_active_admission_ns || submitted.FirstTimestampNs < m_active_admission_ns) + return Fail("invalid runtime admission-to-submission timing"); + const auto cpu = m_cpu_allocator.Statistics(); + const auto staging = m_gpu_allocator.Statistics(); + const auto metadata = m_metadata.GetStatistics(); + const auto gpu = engine->Device->GpuMem.GetAllocationStatistics(); + const auto retirement = engine->Device->PendingFree.GetStatistics(); + ZENGINE_CORE_INFO("[CookedTiming] generation={} admission_ns={} first_submission_ns={} elapsed_ns={}", submitted.Generation, m_active_admission_ns, submitted.FirstTimestampNs, submitted.FirstTimestampNs - m_active_admission_ns) + if (m_observe_gpu_completion) + { + if (m_completion_observed_ns < submitted.FirstTimestampNs) + return Fail("invalid observed render completion timing"); + ZENGINE_CORE_INFO("[CookedGpuCompletion] generation={} observed_ns={} timeline_value={}", submitted.Generation, m_completion_observed_ns, submitted.FirstTimelineValue) + } + ZENGINE_CORE_INFO("[CookedLifecycle] generation={} cpu_live={} cpu_peak={} staging_live={} staging_peak={} metadata_live={} metadata_blocks={} gpu_live={} gpu_peak={} retirement_queued={} retirement_draining={}", submitted.Generation, cpu.LiveBytes, cpu.PeakBytes, staging.LiveBytes, staging.PeakBytes, metadata.LiveBytes, metadata.LiveBlocks, gpu.AllocationBytes, gpu.PeakAllocationBytes, retirement.Queued, retirement.Draining) + const auto domain_peak = [&gpu](Core::Memory::GpuMemoryDomain domain) { return gpu.DomainPeakAdmittedBytes[static_cast(domain)]; }; + ZENGINE_CORE_INFO( + "[CookedGpuBudget] generation={} admitted_peak={} geometry_peak={} texture_peak={} target_peak={} uniform_peak={} staging_peak={} readback_peak={}", submitted.Generation, gpu.PeakAdmittedBytes, domain_peak(Core::Memory::GpuMemoryDomain::DeviceGeometry), domain_peak(Core::Memory::GpuMemoryDomain::DeviceTexture), domain_peak(Core::Memory::GpuMemoryDomain::RenderTarget), domain_peak(Core::Memory::GpuMemoryDomain::HostUniform), domain_peak(Core::Memory::GpuMemoryDomain::HostStaging), domain_peak(Core::Memory::GpuMemoryDomain::HostReadback)) + ZENGINE_CORE_INFO("[CookedRetirementBudget] generation={} capacity={} high_water={}", submitted.Generation, retirement.Capacity, retirement.HighWater) + ZENGINE_CORE_INFO("[CookedCache] generation={} closures={} leases={} retiring={} asset_versions={}", submitted.Generation, cache.Closures, cache.Leases, cache.Retiring, cache.AssetVersions) + m_reported_generation = submitted.Generation; + } + if (m_steady_frames && !m_steady_complete) + { + if (!m_steady_start_ns || m_steady_generation != submitted.Generation || submitted.Count < m_steady_start_count) + { + m_steady_start_count = submitted.Count; + m_steady_generation = submitted.Generation; + m_steady_start_ns = TimestampNs(); + } + if (submitted.Count - m_steady_start_count < m_steady_frames) + return; + const auto end_ns = TimestampNs(); + ZENGINE_CORE_INFO("[CookedSteady] generation={} start_ns={} end_ns={} submitted_frames={}", submitted.Generation, m_steady_start_ns, end_ns, submitted.Count - m_steady_start_count) + m_steady_complete = true; + } + if (m_reloads_remaining) + { + const auto next = m_active_scene == m_initial_scene ? m_alternate_scene : m_initial_scene; + if (m_lifecycle_stress && (m_stress_admitted % 2 == 0)) + { + m_cancel_cpu_probe = m_stress_admitted % 4 == 0; + m_cancel_upload_probe = !m_cancel_cpu_probe; + m_cpu_cancellation_observed = false; + m_stress_cancel_generation = engine->ScenePublication.Generation(); + } + if (RequestScene(next, (m_supersede_waiting_probe || m_supersede_cpu_probe || m_cancel_cpu_probe || (m_lifecycle_stress && m_cancel_upload_probe) || (m_retry_failure_probe && !m_cancel_upload_probe)) ? &m_probe_old : nullptr)) + { + --m_reloads_remaining; + if (m_lifecycle_stress) + ++m_stress_admitted; + if (m_supersede_waiting_probe) + { + m_waiting_superseded_observed = false; + ECS::SceneLoadResult outcome = {}; + if (!RequestScene(m_initial_scene, &m_probe_latest) || !m_waiting_superseded_observed || !PollSceneRequest(m_probe_old, &outcome) || outcome.Outcome != ECS::SceneLoadOutcome::Superseded || outcome.CleanupPending || CancelSceneRequest(m_probe_old) != ECS::SceneCancelResult::AlreadyTerminal || !AcknowledgeSceneRequest(m_probe_old) || PollSceneRequest(m_probe_old, &outcome)) + return Fail("cannot retain superseded waiting request outcome"); + m_probe_old = {}; + m_supersede_waiting_probe = false; + ZENGINE_CORE_INFO("[CookedRuntime] Superseded waiting request before worker admission") + } + } + return; + } + if (m_post_steady_frames && !m_post_steady_complete) + { + if (!m_post_steady_start_ns || m_post_steady_generation != submitted.Generation || submitted.Count < m_post_steady_start_count) + { + m_post_steady_start_count = submitted.Count; + m_post_steady_generation = submitted.Generation; + m_post_steady_start_ns = TimestampNs(); + } + if (submitted.Count - m_post_steady_start_count < m_post_steady_frames) + return; + const auto end_ns = TimestampNs(); + ZENGINE_CORE_INFO("[CookedPostSteady] generation={} start_ns={} end_ns={} submitted_frames={}", submitted.Generation, m_post_steady_start_ns, end_ns, submitted.Count - m_post_steady_start_count) + m_post_steady_complete = true; + } + if (m_capture_path.empty()) + { + if (m_lifecycle_stress) + { + if (m_stress_cancelled + m_stress_committed != m_stress_admitted) + return Fail("stress cycle lost a terminal outcome"); + ZENGINE_CORE_INFO("[CookedStress] admitted={} cancelled={} committed={}", m_stress_admitted, m_stress_cancelled, m_stress_committed) + } + m_frame_recorded = true; + ZENGINE_CORE_INFO("[CookedRuntime] Submitted {} draws", submitted.DrawCount) + Engine::RequestClose(); + } + else + { + const auto capture = m_capture_state.load(std::memory_order_acquire); + if (capture == 0) + { + if (!m_width || !m_height || uint64_t(m_width) * m_height > 32ULL * 1024 * 1024) + return Fail("invalid capture extent"); + m_capture_width = m_width; + m_capture_height = m_height; + m_capture_generation = submitted.Generation; + m_capture_bytes = size_t(m_width) * m_height * 4; + m_capture_pixels.reset(new (std::nothrow) uint8_t[m_capture_bytes]); + if (!m_capture_pixels) + return Fail("cannot allocate capture buffer"); + m_capture_state.store(1, std::memory_order_release); + } + else if (capture == 3) + { + const size_t pixels = size_t(m_capture_width) * m_capture_height; + for (size_t i = 0; i < pixels; ++i) + { + m_capture_pixels[3 * i] = m_capture_pixels[4 * i]; + m_capture_pixels[3 * i + 1] = m_capture_pixels[4 * i + 1]; + m_capture_pixels[3 * i + 2] = m_capture_pixels[4 * i + 2]; + } + std::ofstream output(m_capture_path, std::ios::binary | std::ios::trunc); + output << "P6\n" << m_capture_width << ' ' << m_capture_height << "\n255\n"; + output.write(reinterpret_cast(m_capture_pixels.get()), static_cast(pixels * 3)); + output.close(); + if (output.fail()) + return Fail("cannot write scene capture"); + m_frame_recorded = true; + ZENGINE_CORE_INFO("[CookedRuntime] Submitted {} draws", submitted.DrawCount) + Engine::RequestClose(); + } + else if (capture == 4) + return Fail("scene capture failed"); + } + } + } + } + + void StandaloneRuntimeApplication::PrepareScene(RenderFrameState& state) + { + state.RenderTargetW = m_width; + state.RenderTargetH = m_height; + state.ResizeSequence = m_resize_sequence; + state.ShowGrid = false; + state.Gizmo.Reset(); + state.SceneCapture = {}; + if (m_capture_state.load(std::memory_order_acquire) == 1) + state.SceneCapture = {this, &BeginCapture, &CompleteCapture, &CancelCapture}; + if (!m_host.Active()) + state.Camera = m_bootstrap_camera; + if (auto* active = m_host.Active()) + { + if (m_width && m_height && !ECS::CaptureSceneRuntimeCamera(active->Instance().World(), m_camera, m_width, m_height, &state.Camera, &m_scene_diagnostics)) + return Fail("active runtime camera is invalid"); + state.SceneResources = active->Resources().InitialResources(); + } + } + + void StandaloneRuntimeApplication::OnRenderUI() + { + auto* ctx = RenderPipeline ? RenderPipeline->ZUICtx : nullptr; + if (!ctx || !RenderPipeline->SceneRenderer) + return; + if (ctx->ScreenW && ctx->ScreenH && (ctx->ScreenW != m_width || ctx->ScreenH != m_height)) + { + m_width = ctx->ScreenW; + m_height = ctx->ScreenH; + ++m_resize_sequence; + } + const auto texture = RenderPipeline->SceneRenderer->GetFrameOutput(); + auto* image = UI::ZUIBeginColumn(ctx, "##runtime", UI::ZPx(static_cast(ctx->ScreenW)), UI::ZPx(static_cast(ctx->ScreenH))); + image->Flags = image->Flags | UI::ZUI_DrawBackground; + image->TextureIndex = texture.Valid() ? static_cast(texture.Index) : 0xFFFFFFFFu; + image->EdgeSoftness = 0.f; + ZUIBoxSetColor(image, 1.f, 1.f, 1.f, texture.Valid() ? 1.f : 0.f); + UI::ZUIEndColumn(ctx); + } + + void StandaloneRuntimeApplication::OnClosing() + { + RestoreStagingLimit(false); + m_cancelled.value.store(true, std::memory_order_release); + m_gpu.Cancel(); + } + + void StandaloneRuntimeApplication::OnWorldShutdown(EngineContext& engine) + { + m_gpu.Cancel(); + (void) m_gpu.Tick(); + m_candidate.reset(); + m_gpu.ReleaseTerminalAssets(); + const bool reset = m_gpu.Reset(); + const bool retired = m_host.ShutdownAfterReadersJoined(engine, *this); + ZENGINE_VALIDATE_ASSERT(reset && retired, "Runtime scene shutdown did not drain its owners") + m_assets.Reset(); + m_root.Reset(); + m_storage.Reset(); + if (m_request) + { + if (engine.SceneLoads.Pending(m_request)) + engine.SceneLoads.Cancel(m_request); + ECS::SceneLoadResult shutdown_result = {}; + const bool terminal = !m_shutdown_probe_started || (engine.SceneLoads.Poll(m_request, &shutdown_result) && shutdown_result.Outcome == ECS::SceneLoadOutcome::Cancelled && shutdown_result.CleanupPending && shutdown_result.ExpectedGeneration == m_shutdown_probe_generation && engine.ScenePublication.Generation() == m_shutdown_probe_generation); + const bool cleaned = engine.SceneLoads.FinishCleanup(m_request); + if (m_shutdown_probe_started) + { + m_shutdown_probe_passed = terminal && cleaned && engine.SceneLoads.Poll(m_request, &shutdown_result) && !shutdown_result.CleanupPending && shutdown_result.Outcome == ECS::SceneLoadOutcome::Cancelled; + if (m_shutdown_probe_passed) + ZENGINE_CORE_INFO("[CookedRuntime] Shutdown cancelled and drained pending replacement") + else + ZENGINE_CORE_ERROR("[CookedRuntime] Shutdown replacement outcome or cleanup invalid") + } + if (!m_request_retain_result) + (void) engine.SceneLoads.Acknowledge(m_request); + m_request = {}; + m_request_retain_result = false; + } + if (m_waiting_request) + { + if (engine.SceneLoads.Pending(m_waiting_request)) + (void) engine.SceneLoads.Cancel(m_waiting_request); + if (!m_waiting_retain_result) + (void) engine.SceneLoads.Acknowledge(m_waiting_request); + m_waiting_request = {}; + m_waiting_retain_result = false; + } + if (m_source_audit) + { + const auto vfs = static_cast(engine.VFS)->DeniedSourceAccesses(); + const auto registry = Managers::AssetManager::Instance()->Registry->DeniedLookupCount(); + const auto imports = engine.ImportCoordinator->DeniedImportCount(); + m_source_audit_passed = !vfs && !registry && !imports; + ZENGINE_CORE_INFO("[CookedSourceAudit] vfs_denied={} registry_denied={} imports_denied={}", vfs, registry, imports) + if (!m_source_audit_passed) + ZENGINE_CORE_ERROR("[CookedRuntime] Source-independence audit observed denied access") + } + } + + void StandaloneRuntimeApplication::OnClosed() + { + if (m_cache_initialized) + { + while (m_cache.PumpOne()) + { + } + const bool closed = m_cache.Shutdown(); + ZENGINE_VALIDATE_ASSERT(closed, "Runtime cooked cache still has retained assets") + (void) closed; + } + if (m_vfs_initialized) + m_vfs.Shutdown(); + if (m_metadata.Pool) + m_metadata.Shutdown(); + m_capture_pixels.reset(); + } + bool StandaloneRuntimeApplication::UploadCancellationObserved() const noexcept + { + return m_upload_cancellation_observed; + } + + bool StandaloneRuntimeApplication::CpuCancellationObserved() const noexcept + { + return m_cpu_cancellation_observed; + } + + void StandaloneRuntimeApplication::EnableUploadSubmissionRejectionProbe() noexcept + { + m_reject_upload_probe = true; + } + + void StandaloneRuntimeApplication::EnableStagingRefusalProbe() noexcept + { + m_fail_staging_probe = true; + } + + void StandaloneRuntimeApplication::EnablePreflightRefusalProbe() noexcept + { + m_fail_preflight_probe = true; + } + + void StandaloneRuntimeApplication::EnableDeviceLossProbe() noexcept + { + m_device_loss_probe = true; + } + + void StandaloneRuntimeApplication::EnableSourceIndependenceAudit() noexcept + { + m_source_audit = true; + } + + bool StandaloneRuntimeApplication::WantsSourceIndependenceAudit() const + { + return m_source_audit; + } + + bool StandaloneRuntimeApplication::SourceAuditPassed() const noexcept + { + return m_source_audit_passed; + } + + void StandaloneRuntimeApplication::ConfigureSteadyFrames(uint32_t frames) noexcept + { + m_steady_frames = frames; + } + + void StandaloneRuntimeApplication::ConfigurePostSteadyFrames(uint32_t frames) noexcept + { + m_post_steady_frames = frames; + } + + void StandaloneRuntimeApplication::EnableLifecycleStress() noexcept + { + m_lifecycle_stress = true; + } + + void StandaloneRuntimeApplication::EnableFixedTestWindow() noexcept + { + m_fixed_test_window = true; + } + + void StandaloneRuntimeApplication::EnableGpuCompletionObservation() noexcept + { + m_observe_gpu_completion = true; + } + + void StandaloneRuntimeApplication::EnableShutdownProbe(bool gpu) noexcept + { + m_shutdown_probe_cpu = !gpu; + m_shutdown_probe_gpu = gpu; + } + + bool StandaloneRuntimeApplication::ShutdownProbePassed() const noexcept + { + return m_shutdown_probe_passed; + } + + bool StandaloneRuntimeApplication::StagingRefusalObserved() const noexcept + { + return m_staging_refusal_observed; + } + + bool StandaloneRuntimeApplication::FrameRecorded() const noexcept + { + return m_frame_recorded; + } + + void StandaloneRuntimeApplication::ConfigureCapture(std::string path) + { + m_capture_path = std::move(path); + } + + void StandaloneRuntimeApplication::OnEvent(Core::CoreEvent&) {} + + void StandaloneRuntimeApplication::OnPreRender() {} + + void StandaloneRuntimeApplication::OnPostRender() {} + + bool StandaloneRuntimeApplication::ShouldSimulate() const + { + return false; + } + +} // namespace ZEngine::Applications diff --git a/ZEngine/ZEngine/Applications/StandaloneRuntimeApplication.h b/ZEngine/ZEngine/Applications/StandaloneRuntimeApplication.h new file mode 100644 index 000000000..be7349457 --- /dev/null +++ b/ZEngine/ZEngine/Applications/StandaloneRuntimeApplication.h @@ -0,0 +1,241 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ZEngine::Applications +{ + /// @brief Standalone, renderer-only runtime without an editor session. + /// @details The current loader consumes cooked packages. Gameplay simulation + /// remains disabled until the runtime gameplay services are integrated. + class StandaloneRuntimeApplication final : public GameApplication + { + public: + /// @brief Configure the initial cooked scene and optional lifecycle probes. + /// @details Configure before initialization; test probes are opt-in. + void Configure(std::string directory, uuids::uuid scene, uuids::uuid camera, bool exit_after_frame, uint32_t expected_draws, uint32_t reloads = 0, uuids::uuid alternate = {}, bool retry_failure_probe = false, bool cancel_upload_probe = false, bool cancel_cpu_probe = false, bool supersede_cpu_probe = false, bool supersede_gpu_probe = false, bool supersede_waiting_probe = false); + /// @brief Admit a scene request on the main thread. + /// @details A newer request supersedes older work, which drains before + /// the waiting request starts. An output ID retains its terminal result + /// until acknowledged; refusal preserves the output. + [[nodiscard]] bool RequestScene(const uuids::uuid& scene, ECS::SceneRequestId* output = nullptr); + /// @brief Cancel the most recent pending scene request. + /// @details The outcome is published immediately; cleanup remains pending + /// until the worker and GPU payload have drained. + [[nodiscard]] ECS::SceneCancelResult CancelSceneRequest() noexcept; + /// @brief Cancel the specified pending scene request. + [[nodiscard]] ECS::SceneCancelResult CancelSceneRequest(ECS::SceneRequestId request) noexcept; + /// @brief Read the retained outcome for a scene request. + [[nodiscard]] bool PollSceneRequest(ECS::SceneRequestId request, ECS::SceneLoadResult* output) const noexcept; + /// @brief Release a retained terminal request outcome. + [[nodiscard]] bool AcknowledgeSceneRequest(ECS::SceneRequestId request) noexcept; + /// @brief Report whether the upload cancellation probe reached its boundary. + [[nodiscard]] bool UploadCancellationObserved() const noexcept; + /// @brief Report whether the CPU cancellation probe reached its boundary. + [[nodiscard]] bool CpuCancellationObserved() const noexcept; + /// @brief Inject a rejection before a recorded upload is submitted. + void EnableUploadSubmissionRejectionProbe() noexcept; + /// @brief Inject a replacement staging-budget refusal. + void EnableStagingRefusalProbe() noexcept; + /// @brief Inject a refusal before publication of a prepared replacement. + void EnablePreflightRefusalProbe() noexcept; + /// @brief Inject device loss during the upload probe. + void EnableDeviceLossProbe() noexcept; + /// @brief Enable the standalone source-access audit. + void EnableSourceIndependenceAudit() noexcept; + /// @brief Report whether engine services should enforce the source audit. + [[nodiscard]] bool WantsSourceIndependenceAudit() const override; + /// @brief Report the final source-access audit result after shutdown. + [[nodiscard]] bool SourceAuditPassed() const noexcept; + /// @brief Configure steady frames to observe before replacement. + void ConfigureSteadyFrames(uint32_t frames) noexcept; + /// @brief Configure steady frames to observe after replacement. + void ConfigurePostSteadyFrames(uint32_t frames) noexcept; + /// @brief Enable the alternating cancellation and replacement probe. + void EnableLifecycleStress() noexcept; + /// @brief Use the fixed-size window required by test captures. + void EnableFixedTestWindow() noexcept; + /// @brief Observe GPU completion of submitted scene frames. + void EnableGpuCompletionObservation() noexcept; + /// @brief Request shutdown while CPU or GPU preparation is pending. + void EnableShutdownProbe(bool gpu) noexcept; + /// @brief Report whether the shutdown probe drained its pending request. + [[nodiscard]] bool ShutdownProbePassed() const noexcept; + /// @brief Report whether the staging refusal probe observed its failure. + [[nodiscard]] bool StagingRefusalObserved() const noexcept; + /// @brief Report whether the requested scene frame was recorded. + [[nodiscard]] bool FrameRecorded() const noexcept; + /// @brief Set the output path for the optional scene-target capture. + void ConfigureCapture(std::string path); + /// @brief Apply the optional fixed test-window configuration. + void OverrideWindowConfiguration() override; + /// @brief Configure application state before engine initialization. + void OnInitializing() override; + /// @brief Initialize runtime services and admit the first scene. + void OnInitialized() override; + /// @brief Ignore editor events in the standalone renderer. + void OnEvent(Core::CoreEvent&) override; + /// @brief Advance scene admission, reconstruction and publication. + void OnUpdate(float) override; + /// @brief No standalone pre-render hook is required. + void OnPreRender() override; + /// @brief No standalone post-render hook is required. + void OnPostRender() override; + /// @brief Display the runtime scene output in a full-window view. + void OnRenderUI() override; + /// @brief Populate the render frame from the active runtime scene. + void PrepareScene(RenderFrameState&) override; + /// @brief Begin cancellation and cleanup when the application closes. + void OnClosing() override; + /// @brief Release application-owned allocations after engine shutdown. + void OnClosed() override; + /// @brief Retire scene owners after engine readers have joined. + void OnWorldShutdown(EngineContext&) override; + /// @brief Keep simulation disabled for the renderer-only runtime. + [[nodiscard]] bool ShouldSimulate() const override; + + private: + /// @brief Reconstruct the admitted cooked scene on a worker. + static void RunWorker(void*); + /// @brief Drain queued cooked cache work on a worker. + static void DrainCacheWorker(void*); + /// @brief Begin an optional render-target capture. + static bool BeginCapture(void*, uint32_t, uint32_t, uint64_t); + /// @brief Complete a render-target capture with its readback bytes. + static void CompleteCapture(const void*, size_t, void*); + /// @brief Cancel an outstanding render-target capture. + static void CancelCapture(void*); + /// @brief Start the newest waiting request once its resources are available. + void StartWaiting(); + /// @brief Restore the staging budget after a refusal probe. + void RestoreStagingLimit(bool request_failed) noexcept; + /// @brief Fail the current request and preserve the active scene when possible. + void Fail(const char* message, ECS::SceneLoadStage stage = ECS::SceneLoadStage::Preparation); + Core::Memory::ArenaAllocator m_arena = {}; + Cook::CookPageAllocator m_cpu_allocator = {}; + Cook::CookPageAllocator m_gpu_allocator = {}; + Cook::CookCodecAllocationRoute m_cpu_route{m_cpu_allocator}; + Cook::CookCodecAllocationRoute m_gpu_route{m_gpu_allocator}; + Core::Memory::TLSFSlab m_metadata = {}; + Core::VFS::VFSDiskBackend m_backend = {}; + Core::VFS::VFSContext m_vfs = {}; + Core::VFS::VFSReadRootLease m_root = {}; + Managers::CookedCpuClosureCache m_cache = {}; + ECS::SceneStorageLease m_storage = {}; + Rendering::Scenes::RenderScene m_bootstrap = {}; + Rendering::Cameras::CameraFrameData m_bootstrap_camera = {}; + std::unique_ptr m_candidate = {}; + Managers::CookedCpuClosureCache::Lease m_assets = {}; + Rendering::Scenes::CookedSceneResourceRequest m_gpu = {}; + RuntimeSceneHost m_host = {}; + Cook::CookDiagnostics m_cook_diagnostics = {}; + ECS::SceneDiagnostics m_scene_diagnostics = {}; + CookedRuntimeReconstructionError m_worker_result = CookedRuntimeReconstructionError::InvalidInput; + /// @brief Isolate worker handoff flags from independently written state. + PaddedAtomic m_cancelled = {}; + PaddedAtomic m_worker_done = {}; + PaddedAtomic m_cache_drain_running = {}; + std::atomic m_cpu_boundary = false; + std::string m_directory = {}; + std::string m_capture_path = {}; + std::unique_ptr m_capture_pixels = {}; + std::atomic m_capture_state = 0; + size_t m_capture_bytes = 0; + uint32_t m_capture_width = 0; + uint32_t m_capture_height = 0; + uint64_t m_capture_generation = 0; + uuids::uuid m_scene = {}; + uuids::uuid m_initial_scene = {}; + uuids::uuid m_active_scene = {}; + uuids::uuid m_alternate_scene = {}; + uuids::uuid m_camera = {}; + ECS::SceneRequestId m_request = {}; + ECS::SceneRequestId m_waiting_request = {}; + bool m_request_retain_result = false; + bool m_waiting_retain_result = false; + ECS::SceneLoadOutcome m_request_terminal = ECS::SceneLoadOutcome::Pending; + ECS::SceneRequestId m_probe_old = {}; + ECS::SceneRequestId m_probe_latest = {}; + uuids::uuid m_waiting_scene = {}; + uint64_t m_waiting_generation = 0; + uint64_t m_waiting_admission_ns = 0; + uint64_t m_generation = 0; + uint64_t m_admission_ns = 0; + uint64_t m_active_admission_ns = 0; + uint64_t m_completed_generation = 0; + uint64_t m_completion_observed_ns = 0; + bool m_observe_gpu_completion = false; + uint64_t m_reported_generation = 0; + uint64_t m_steady_start_ns = 0; + uint64_t m_steady_start_count = 0; + uint64_t m_steady_generation = 0; + uint32_t m_steady_frames = 0; + bool m_steady_complete = false; + uint64_t m_post_steady_start_ns = 0; + uint64_t m_post_steady_start_count = 0; + uint64_t m_post_steady_generation = 0; + uint32_t m_post_steady_frames = 0; + bool m_post_steady_complete = false; + uint64_t m_resize_sequence = 1; + uint32_t m_width = 0; + uint32_t m_height = 0; + uint32_t m_expected_draws = 0; + uint32_t m_reloads_remaining = 0; + uint32_t m_stress_admitted = 0; + uint32_t m_stress_cancelled = 0; + uint32_t m_stress_committed = 0; + uint64_t m_stress_cancel_generation = 0; + uint64_t m_stress_cancel_requested_ns = 0; + uint64_t m_stress_submission_baseline = 0; + bool m_lifecycle_stress = false; + bool m_fixed_test_window = false; + bool m_shutdown_probe_cpu = false; + bool m_shutdown_probe_gpu = false; + bool m_shutdown_probe_started = false; + bool m_shutdown_probe_passed = false; + uint64_t m_shutdown_probe_generation = 0; + bool m_stress_waiting_frame = false; + uint8_t m_last_gpu_status = 255; + bool m_publication_wait_logged = false; + bool m_exit_after_frame = false; + bool m_frame_recorded = false; + bool m_initialized = false; + bool m_vfs_initialized = false; + bool m_cache_initialized = false; + bool m_dispatched = false; + bool m_gpu_started = false; + bool m_failed = false; + bool m_retry_failure_probe = false; + bool m_cancel_upload_probe = false; + bool m_upload_cancellation_observed = false; + bool m_cancel_cpu_probe = false; + bool m_cpu_pause_armed = false; + bool m_cpu_cancellation_observed = false; + bool m_supersede_cpu_probe = false; + bool m_supersede_gpu_probe = false; + bool m_supersede_waiting_probe = false; + bool m_reject_upload_probe = false; + bool m_device_loss_probe = false; + bool m_source_audit = false; + bool m_source_audit_passed = false; + bool m_fail_staging_probe = false; + bool m_fail_preflight_probe = false; + bool m_staging_limit_active = false; + bool m_staging_gpu_started = false; + bool m_staging_refusal_observed = false; + uint64_t m_staging_refusal_baseline = 0; + Core::Memory::GpuAllocationLimits m_saved_gpu_limits = {}; + bool m_waiting_superseded_observed = false; + bool m_cpu_supersede_armed = false; + bool m_waiting_failure_frame = false; + uint64_t m_failure_submission_baseline = 0; + }; +} // namespace ZEngine::Applications diff --git a/ZEngine/ZEngine/Core/Containers/SPSCQueue.h b/ZEngine/ZEngine/Core/Containers/SPSCQueue.h index c960fd504..2e500309d 100644 --- a/ZEngine/ZEngine/Core/Containers/SPSCQueue.h +++ b/ZEngine/ZEngine/Core/Containers/SPSCQueue.h @@ -49,6 +49,14 @@ namespace ZEngine::Core::Containers return m_tail.value.load(std::memory_order_acquire) == m_head.value.load(std::memory_order_acquire); } + // Producer-only admission check. A concurrent consumer can only make + // more room; another producer must not enqueue between check and push. + bool full() const + { + const uint32_t head = m_head.value.load(std::memory_order_relaxed); + return ((head + 1) & MASK) == m_tail.value.load(std::memory_order_acquire); + } + private: PaddedAtomic m_head{}; PaddedAtomic m_tail{}; diff --git a/ZEngine/ZEngine/Core/Memory/GpuAllocator.cpp b/ZEngine/ZEngine/Core/Memory/GpuAllocator.cpp index e4d5fe693..2b102f63e 100644 --- a/ZEngine/ZEngine/Core/Memory/GpuAllocator.cpp +++ b/ZEngine/ZEngine/Core/Memory/GpuAllocator.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #ifdef VMA_DEBUG_DETECT_CORRUPTION #include @@ -28,6 +30,36 @@ namespace ZEngine::Core::Memory { + namespace + { + // VMA may round the requested backing size even when the legacy Vulkan + // requirement size is unaligned. Reserve a conservative aligned amount + // before asking it for memory, including host atom and small-granularity + // rounding. Commit still checks the size VMA actually returned. + VkDeviceSize AdmissionBytes(VmaAllocator allocator, const VkMemoryRequirements& requirements) + { + const VkPhysicalDeviceProperties* properties = nullptr; + vmaGetPhysicalDeviceProperties(allocator, &properties); + if (!properties || !requirements.size || !requirements.alignment) + { + return 0; + } + VkDeviceSize alignment = std::max(requirements.alignment, properties->limits.nonCoherentAtomSize); + const auto granularity = properties->limits.bufferImageGranularity; + if (granularity <= 256) + { + alignment = std::max(alignment, granularity); + } + const VkDeviceSize remainder = requirements.size % alignment; + const VkDeviceSize padding = remainder ? alignment - remainder : 0; + if (requirements.size > std::numeric_limits::max() - padding) + { + return 0; + } + return requirements.size + padding; + } + } // namespace + void GpuAllocator::Initialize(VkPhysicalDevice physical_device, VkDevice device, VkInstance instance, bool has_memory_budget_ext, bool has_buffer_device_address_ext) { m_allocations_initialized = false; @@ -298,7 +330,7 @@ namespace ZEngine::Core::Memory { VkMemoryRequirements requirements = {}; vkGetBufferMemoryRequirements(allocator_info.device, buffer_view.Handle, &requirements); - if (!AdmitAllocationBytes(record, requirements.size, domain)) + if (!AdmitAllocationBytes(record, AdmissionBytes(Allocator, requirements), domain)) { result = VK_ERROR_OUT_OF_DEVICE_MEMORY; } @@ -348,7 +380,16 @@ namespace ZEngine::Core::Memory ReleaseAllocation(record, nullptr); return {}; } - CommitAllocation(record, buffer_view.Allocation); + if (!CommitAllocation(record, buffer_view.Allocation)) + { + vmaDestroyBuffer(Allocator, buffer_view.Handle, buffer_view.Allocation); + ReleaseAllocation(record, nullptr); + if (status) + { + *status = VK_ERROR_OUT_OF_DEVICE_MEMORY; + } + return {}; + } vmaSetAllocationName(Allocator, buffer_view.Allocation, debug_name); buffer_view.MappedData = allocation_info.pMappedData; buffer_view.RetirementReservation = retirement.Detach(); @@ -451,16 +492,19 @@ namespace ZEngine::Core::Memory return index; } - void GpuAllocator::CommitAllocation(uint32_t index, VmaAllocation allocation) + bool GpuAllocator::CommitAllocation(uint32_t index, VmaAllocation allocation) { const auto bytes = GetAllocationSize(allocation); std::lock_guard lock(m_allocation_mutex); ZENGINE_VALIDATE_ASSERT(index < AllocationCapacity && allocation, "Invalid GPU allocation commit") auto& record = m_allocations[index]; ZENGINE_VALIDATE_ASSERT(record.References == 1 && !record.Allocation, "GPU allocation already committed") + if (!bytes || bytes > record.Bytes) + { + return false; + } record.Allocation = allocation; - ZENGINE_VALIDATE_ASSERT(bytes <= record.Bytes, "GPU allocation exceeds admitted native requirements") - record.LiveBytes = bytes; + record.LiveBytes = bytes; ++m_allocation_statistics.AllocationCount; m_allocation_statistics.AllocationBytes += bytes; if (m_allocation_statistics.AllocationCount > m_allocation_statistics.PeakAllocationCount) @@ -471,6 +515,7 @@ namespace ZEngine::Core::Memory { m_allocation_statistics.PeakAllocationBytes = m_allocation_statistics.AllocationBytes; } + return true; } bool GpuAllocator::RetainAllocation(uint32_t index, VmaAllocation allocation) @@ -527,14 +572,26 @@ namespace ZEngine::Core::Memory ZENGINE_VALIDATE_ASSERT(record.References == 1 && !record.Allocation && !record.Bytes, "GPU allocation already charged") const auto domain_index = static_cast(domain); auto& domain_bytes = m_allocation_statistics.DomainAdmittedBytes[domain_index]; - if (!bytes || bytes > m_allocation_limits.TotalBytes - m_allocation_statistics.AdmittedBytes || bytes > m_allocation_limits.DomainBytes[domain_index] - domain_bytes) + if (!bytes) { return false; } + if (bytes > m_allocation_limits.TotalBytes - m_allocation_statistics.AdmittedBytes || bytes > m_allocation_limits.DomainBytes[domain_index] - domain_bytes) + { + auto& refusals = m_allocation_statistics.DomainBudgetRefusals[domain_index]; + if (refusals != std::numeric_limits::max()) + ++refusals; + return false; + } record.Bytes = bytes; record.Domain = domain; m_allocation_statistics.AdmittedBytes += bytes; domain_bytes += bytes; + auto& domain_peak = m_allocation_statistics.DomainPeakAdmittedBytes[domain_index]; + if (domain_bytes > domain_peak) + { + domain_peak = domain_bytes; + } if (m_allocation_statistics.AdmittedBytes > m_allocation_statistics.PeakAdmittedBytes) { m_allocation_statistics.PeakAdmittedBytes = m_allocation_statistics.AdmittedBytes; @@ -621,7 +678,7 @@ namespace ZEngine::Core::Memory { VkMemoryRequirements requirements = {}; vkGetImageMemoryRequirements(device, buffer_image.Handle, &requirements); - if (!AdmitAllocationBytes(record, requirements.size, domain)) + if (!AdmitAllocationBytes(record, AdmissionBytes(Allocator, requirements), domain)) { result = VK_ERROR_OUT_OF_DEVICE_MEMORY; } @@ -668,7 +725,16 @@ namespace ZEngine::Core::Memory } return {}; } - CommitAllocation(record, buffer_image.Allocation); + if (!CommitAllocation(record, buffer_image.Allocation)) + { + vmaDestroyImage(Allocator, buffer_image.Handle, buffer_image.Allocation); + ReleaseAllocation(record, nullptr); + if (status) + { + *status = VK_ERROR_OUT_OF_DEVICE_MEMORY; + } + return {}; + } vmaSetAllocationName(Allocator, buffer_image.Allocation, debug_name); result = CreateDefaultImageView(device, image_info, buffer_image.Handle, aspect, view_type, layer_count, &buffer_image.ViewHandle); diff --git a/ZEngine/ZEngine/Core/Memory/GpuAllocator.h b/ZEngine/ZEngine/Core/Memory/GpuAllocator.h index 9e3947ade..8778c0759 100644 --- a/ZEngine/ZEngine/Core/Memory/GpuAllocator.h +++ b/ZEngine/ZEngine/Core/Memory/GpuAllocator.h @@ -163,15 +163,17 @@ namespace ZEngine::Core::Memory // Includes the staging ring and one-shot staging allocations. struct GpuAllocationStatistics { - uint32_t Capacity = 0; - uint32_t ReservedRecords = 0; - uint32_t AllocationCount = 0; - uint32_t PeakAllocationCount = 0; - uint64_t AllocationBytes = 0; - uint64_t PeakAllocationBytes = 0; - uint64_t AdmittedBytes = 0; - uint64_t PeakAdmittedBytes = 0; - uint64_t DomainAdmittedBytes[static_cast(GpuMemoryDomain::Count)] = {}; + uint32_t Capacity = 0; + uint32_t ReservedRecords = 0; + uint32_t AllocationCount = 0; + uint32_t PeakAllocationCount = 0; + uint64_t AllocationBytes = 0; + uint64_t PeakAllocationBytes = 0; + uint64_t AdmittedBytes = 0; + uint64_t PeakAdmittedBytes = 0; + uint64_t DomainAdmittedBytes[static_cast(GpuMemoryDomain::Count)] = {}; + uint64_t DomainPeakAdmittedBytes[static_cast(GpuMemoryDomain::Count)] = {}; + uint64_t DomainBudgetRefusals[static_cast(GpuMemoryDomain::Count)] = {}; }; struct GpuAllocator @@ -245,7 +247,7 @@ namespace ZEngine::Core::Memory uint32_t ReserveAllocation(); bool AdmitAllocationBytes(uint32_t record, VkDeviceSize bytes, GpuMemoryDomain domain); BufferView TryAllocateBufferWithPolicy(VkDeviceSize size, VkBufferUsageFlags usage, GpuMemoryDomain domain, const char* debug_name, VkResult* status, bool prefer_device); - void CommitAllocation(uint32_t record, VmaAllocation allocation); + bool CommitAllocation(uint32_t record, VmaAllocation allocation); bool RetainAllocation(uint32_t record, VmaAllocation allocation); void ReleaseAllocation(uint32_t record, VmaAllocation allocation); }; diff --git a/ZEngine/ZEngine/Core/VFS/Registry/AssetRegistry.cpp b/ZEngine/ZEngine/Core/VFS/Registry/AssetRegistry.cpp index 173104cf3..97389c57d 100644 --- a/ZEngine/ZEngine/Core/VFS/Registry/AssetRegistry.cpp +++ b/ZEngine/ZEngine/Core/VFS/Registry/AssetRegistry.cpp @@ -46,26 +46,36 @@ namespace ZEngine::Core::VFS bool AssetRegistry::TryReadRecord(const uuids::uuid& uuid, AssetRecordSnapshot* out) const { + if (DenyLookup()) + return false; return m_index.TryReadRecord(uuid, out); } bool AssetRegistry::TryReadRecordByPath(const Core::VFS::VFSPath& path, AssetRecordSnapshot* out) const { + if (DenyLookup()) + return false; return m_index.TryReadRecordByPath(path, out); } bool AssetRegistry::TryReadSlot(const uuids::uuid& uuid, Managers::AssetType type, Managers::AssetHandle* out) const { + if (DenyLookup()) + return false; return m_index.TryReadSlot(uuid, type, out); } bool AssetRegistry::TryBindLoadedAsset(const AssetRecordSnapshot& expected, Managers::AssetHandle slot) { + if (DenyLookup()) + return false; return m_index.TryBindLoadedAsset(expected, slot); } bool AssetRegistry::TryBindLoadedMesh(const AssetRecordSnapshot& expected, const uuids::uuid& hierarchy, Managers::AssetHandle mesh_slot, Managers::AssetHandle hierarchy_slot, const AssetRecordSnapshot* retained_hierarchy) { + if (DenyLookup()) + return false; return m_index.TryBindLoadedMesh(expected, hierarchy, mesh_slot, hierarchy_slot, retained_hierarchy); } @@ -140,30 +150,40 @@ namespace ZEngine::Core::VFS AssetRecord* AssetRegistry::FindByUUID(const uuids::uuid& uuid) { + if (DenyLookup()) + return nullptr; Helpers::Handle h = m_index.FindByUUID(uuid); return h.Valid() ? m_index.Access(h) : nullptr; } const AssetRecord* AssetRegistry::FindByUUID(const uuids::uuid& uuid) const { + if (DenyLookup()) + return nullptr; Helpers::Handle h = m_index.FindByUUID(uuid); return h.Valid() ? m_index.Access(h) : nullptr; } AssetRecord* AssetRegistry::FindByPath(const Core::VFS::VFSPath& path) { + if (DenyLookup()) + return nullptr; Helpers::Handle h = m_index.FindByPath(path); return h.Valid() ? m_index.Access(h) : nullptr; } const AssetRecord* AssetRegistry::FindByPath(const Core::VFS::VFSPath& path) const { + if (DenyLookup()) + return nullptr; Helpers::Handle h = m_index.FindByPath(path); return h.Valid() ? m_index.Access(h) : nullptr; } AssetRecord* AssetRegistry::Access(Helpers::Handle handle) { + if (DenyLookup()) + return nullptr; return handle.Valid() ? m_index.Access(handle) : nullptr; } @@ -504,6 +524,8 @@ namespace ZEngine::Core::VFS { QueryResult result{}; result.Handles.init(arena, 64); + if (DenyLookup()) + return result; if (filter.Type.has_value()) { diff --git a/ZEngine/ZEngine/Core/VFS/Registry/AssetRegistry.h b/ZEngine/ZEngine/Core/VFS/Registry/AssetRegistry.h index e337bb27a..4df86549b 100644 --- a/ZEngine/ZEngine/Core/VFS/Registry/AssetRegistry.h +++ b/ZEngine/ZEngine/Core/VFS/Registry/AssetRegistry.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,14 @@ namespace ZEngine::Core::VFS AssetRegistry() = default; ~AssetRegistry() = default; + void EnableLookupDenialAudit() noexcept + { + m_deny_lookup.store(true, std::memory_order_release); + } + [[nodiscard]] uint64_t DeniedLookupCount() const noexcept + { + return m_denied_lookups.load(std::memory_order_acquire); + } // arena: persistent arena. scratch_size: byte size of BFS scratch sub-arena. void Initialize(Core::Memory::ArenaAllocator* arena, uint64_t scratch_size = 65536); @@ -127,9 +136,18 @@ namespace ZEngine::Core::VFS static bool ExtensionMatches(const Core::VFS::VFSPath& path, const char* ext); private: - AssetIndex m_index = {}; - DependencyGraph m_graph = {}; - void* m_reload_cb_ctx = nullptr; + [[nodiscard]] bool DenyLookup() const noexcept + { + if (!m_deny_lookup.load(std::memory_order_acquire)) + return false; + m_denied_lookups.fetch_add(1, std::memory_order_relaxed); + return true; + } + std::atomic m_deny_lookup = false; + mutable std::atomic m_denied_lookups = 0; + AssetIndex m_index = {}; + DependencyGraph m_graph = {}; + void* m_reload_cb_ctx = nullptr; void (*m_reload_cb)(void*, std::span) = nullptr; void* m_ready_cb_ctx = nullptr; diff --git a/ZEngine/ZEngine/Core/VFS/VFSContext.cpp b/ZEngine/ZEngine/Core/VFS/VFSContext.cpp index ae3159f6a..086bb6c69 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSContext.cpp +++ b/ZEngine/ZEngine/Core/VFS/VFSContext.cpp @@ -58,16 +58,22 @@ namespace ZEngine::Core::VFS VFSResult VFSContext::AcquirePublicationRoot(const VFSPath& root, VFSPublicationRootLease* output, IVFSWorkingMemoryBudget* budget) { + if (DenySourceAccess(root)) + return VFSResult::Fail(VFSError::PermissionDenied); return m_mount_table.AcquirePublicationRoot(root, output, budget); } VFSResult VFSContext::AcquireScratchRoot(const VFSPath& root, VFSScratchRootLease* output, IVFSWorkingMemoryBudget* budget) { + if (DenySourceAccess(root)) + return VFSResult::Fail(VFSError::PermissionDenied); return m_mount_table.AcquireScratchRoot(root, output, budget); } VFSResult VFSContext::AcquireReadRoot(const VFSPath& root, VFSReadRootLease* output, IVFSWorkingMemoryBudget* budget) { + if (DenySourceAccess(root)) + return VFSResult::Fail(VFSError::PermissionDenied); return m_mount_table.AcquireReadRoot(root, output, budget); } @@ -577,6 +583,8 @@ namespace ZEngine::Core::VFS VFSResult VFSContext::Open(const VFSPath& absolute_path, VFSOpenFlags flags) { + if (DenySourceAccess(absolute_path)) + return VFSResult::Fail(VFSError::PermissionDenied); VFSResult resolved = m_mount_table.Resolve(absolute_path); if (resolved.Failed()) { @@ -602,6 +610,8 @@ namespace ZEngine::Core::VFS VFSResult VFSContext::Stat(const VFSPath& absolute_path) { + if (DenySourceAccess(absolute_path)) + return VFSResult::Fail(VFSError::PermissionDenied); VFSResult resolved = m_mount_table.Resolve(absolute_path); if (resolved.Failed()) { @@ -614,6 +624,8 @@ namespace ZEngine::Core::VFS VFSResult VFSContext::Exists(const VFSPath& absolute_path) { + if (DenySourceAccess(absolute_path)) + return VFSResult::Fail(VFSError::PermissionDenied); VFSResult resolved = m_mount_table.Resolve(absolute_path); if (resolved.Failed()) { @@ -627,6 +639,8 @@ namespace ZEngine::Core::VFS VFSResult> VFSContext::List(const VFSPath& absolute_dir, Memory::ArenaAllocator* out_arena) { using ResultT = VFSResult>; + if (DenySourceAccess(absolute_dir)) + return ResultT::Fail(VFSError::PermissionDenied); std::lock_guard arena_lock(m_arena_mutex); auto scratch = ZGetScratch(m_arena); diff --git a/ZEngine/ZEngine/Core/VFS/VFSContext.h b/ZEngine/ZEngine/Core/VFS/VFSContext.h index 9e9f08f7c..714a534fd 100644 --- a/ZEngine/ZEngine/Core/VFS/VFSContext.h +++ b/ZEngine/ZEngine/Core/VFS/VFSContext.h @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace ZEngine @@ -33,6 +34,17 @@ namespace ZEngine::Core::VFS struct VFSContext : IVFSContext { + /// Standalone source-independence probe: only the engine-owned VFS + /// namespace remains readable through this context. The separately + /// mounted cooked package uses its own context. + void EnableEngineOnlyAudit() noexcept + { + m_engine_only_audit.store(true, std::memory_order_release); + } + [[nodiscard]] uint64_t DeniedSourceAccesses() const noexcept + { + return m_denied_source_accesses.load(std::memory_order_acquire); + } void Initialize(Memory::ArenaAllocator* arena, size_t mount_table_capacity = 16); [[nodiscard]] VFSResult NativeStorage() const noexcept override; @@ -78,6 +90,15 @@ namespace ZEngine::Core::VFS void Shutdown() override; private: + [[nodiscard]] bool DenySourceAccess(const VFSPath& path) const noexcept + { + if (!m_engine_only_audit.load(std::memory_order_acquire) || (path.ComponentCount() && path.ComponentAt(0).Equals("ZodiacEngine"))) + return false; + m_denied_source_accesses.fetch_add(1, std::memory_order_relaxed); + return true; + } + std::atomic m_engine_only_audit = false; + mutable std::atomic m_denied_source_accesses = 0; friend struct VFSContextTestAccess; struct ImportPublicationState diff --git a/ZEngine/ZEngine/ECS/CookedSceneReconstruction.cpp b/ZEngine/ZEngine/ECS/CookedSceneReconstruction.cpp new file mode 100644 index 000000000..a47ec7114 --- /dev/null +++ b/ZEngine/ZEngine/ECS/CookedSceneReconstruction.cpp @@ -0,0 +1,53 @@ +#include +#include +#include + +namespace ZEngine::ECS +{ + CookedSceneReconstructionError ReconstructCookedScene(Managers::CookedPackageSnapshot& package, SceneStorageLease&& storage, const SceneComponentSchemaRegistry& schemas, Core::Memory::IAllocationRoute& scene_object_route, size_t max_scene_object_bytes, size_t document_working_bytes, Managers::CookedCpuClosureCache& cache, CookedSceneReconstruction* output, SceneDiagnostics* diagnostics, const SceneDocumentLimits& limits, const std::atomic* cancelled) + { + const auto is_cancelled = [&] { return cancelled && cancelled->load(std::memory_order_acquire); }; + if (!output || output->Instance || output->Assets || !package.IsValid() || package.Schemas() != &schemas || !max_scene_object_bytes || !document_working_bytes || !storage.Arena()) + { + return CookedSceneReconstructionError::InvalidInput; + } + if (is_cancelled()) + { + return CookedSceneReconstructionError::Cancelled; + } + Managers::CookedObjectSnapshot scene = {}; + const auto capture = package.CaptureScene(scene_object_route, max_scene_object_bytes, &scene, cancelled); + if (capture.Failed()) + { + return capture.Error() == Core::VFS::VFSError::Cancelled ? CookedSceneReconstructionError::Cancelled : CookedSceneReconstructionError::SceneObject; + } + CookedSceneReconstruction candidate = {}; + if (!SceneInstance::Create(std::move(storage), &candidate.Instance, diagnostics)) + { + return CookedSceneReconstructionError::ScenePopulation; + } + if (!candidate.Instance->ReadVerifiedCooked(package, scene, schemas, document_working_bytes, diagnostics, limits, cancelled)) + { + return is_cancelled() ? CookedSceneReconstructionError::Cancelled : CookedSceneReconstructionError::SceneDecode; + } + scene.Reset(); + if (is_cancelled()) + { + return CookedSceneReconstructionError::Cancelled; + } + if (!candidate.Instance->PopulateCooked(schemas, diagnostics)) + { + return CookedSceneReconstructionError::ScenePopulation; + } + if (is_cancelled()) + { + return CookedSceneReconstructionError::Cancelled; + } + if (!cache.Request(std::move(package), &candidate.Assets)) + { + return CookedSceneReconstructionError::AssetAdmission; + } + *output = std::move(candidate); + return CookedSceneReconstructionError::None; + } +} // namespace ZEngine::ECS diff --git a/ZEngine/ZEngine/ECS/CookedSceneReconstruction.h b/ZEngine/ZEngine/ECS/CookedSceneReconstruction.h new file mode 100644 index 000000000..684ae0558 --- /dev/null +++ b/ZEngine/ZEngine/ECS/CookedSceneReconstruction.h @@ -0,0 +1,33 @@ +#pragma once +#include +#include + +namespace ZEngine::ECS +{ + /// Exclusive worker-owned unpublished result. The closure lease can be + /// pending; its owner must pump the cache and observe Ready before using + /// the assets. The cache and all its documented dependencies outlive it. + struct CookedSceneReconstruction final + { + std::unique_ptr Instance = {}; + Managers::CookedCpuClosureCache::Lease Assets = {}; + }; + + enum class CookedSceneReconstructionError : uint8_t + { + None, + InvalidInput, + SceneObject, + SceneDecode, + ScenePopulation, + AssetAdmission, + Cancelled, + }; + + /// Worker transaction. Scene bytes use the supplied bounded route and are + /// released before return; the document uses the admitted scene slot. The + /// package is moved into the cache on a new closure and remains valid on a + /// cache hit. Failure preserves output and package, but may consume and + /// release storage after SceneInstance::Create. No active scene is changed. + [[nodiscard]] CookedSceneReconstructionError ReconstructCookedScene(Managers::CookedPackageSnapshot& package, SceneStorageLease&& storage, const SceneComponentSchemaRegistry& schemas, Core::Memory::IAllocationRoute& scene_object_route, size_t max_scene_object_bytes, size_t document_working_bytes, Managers::CookedCpuClosureCache& cache, CookedSceneReconstruction* output, SceneDiagnostics* diagnostics, const SceneDocumentLimits& limits = {}, const std::atomic* cancelled = nullptr); +} // namespace ZEngine::ECS diff --git a/ZEngine/ZEngine/ECS/SceneInstance.cpp b/ZEngine/ZEngine/ECS/SceneInstance.cpp index 1e309248a..c451f7387 100644 --- a/ZEngine/ZEngine/ECS/SceneInstance.cpp +++ b/ZEngine/ZEngine/ECS/SceneInstance.cpp @@ -1,4 +1,6 @@ +#include #include +#include #include #include #include @@ -60,7 +62,7 @@ namespace ZEngine::ECS bool SceneInstance::Populate(SceneDocument&& document, const SceneComponentSchemaRegistry& schemas, SceneDiagnostics* diagnostics) { - if (m_source_read_attempted) + if (m_owned_document_kind != OwnedDocumentKind::None) { SceneDiagnosticError(diagnostics, "An instance-owned source document cannot be replaced by an external snapshot.", SceneDiagnosticCode::InvalidArgument); return false; @@ -97,12 +99,12 @@ namespace ZEngine::ECS bool SceneInstance::ReadSource(Core::VFS::IVFSContext& vfs, const Core::VFS::VFSPath& path, const SceneComponentSchemaRegistry& schemas, size_t working_bytes, SceneDiagnostics* diagnostics, const SceneDocumentLimits& limits, const SceneMigrationRegistry* migrations) { ZENGINE_PROFILE_SCOPE("SceneLoad.Worker.SourceRead"); - if (!m_initialized || m_source_read_attempted || m_population_attempted || m_document.Settings() || m_world.AliveCount() || !working_bytes) + if (!m_initialized || m_owned_document_kind != OwnedDocumentKind::None || m_population_attempted || m_document.Settings() || m_world.AliveCount() || !working_bytes) { SceneDiagnosticError(diagnostics, "Source reconstruction requires a fresh owner and a finite document workspace.", SceneDiagnosticCode::InvalidArgument); return false; } - m_source_read_attempted = true; + m_owned_document_kind = OwnedDocumentKind::Source; if (!m_document_route.Initialize(Arena(), working_bytes)) { SceneDiagnosticError(diagnostics, "Cannot reserve the document workspace within admitted scene storage.", SceneDiagnosticCode::AllocationFailed); @@ -116,7 +118,7 @@ namespace ZEngine::ECS bool SceneInstance::PopulateSource(const SceneComponentSchemaRegistry& schemas, SceneDiagnostics* diagnostics) { ZENGINE_PROFILE_SCOPE("SceneLoad.Worker.Populate"); - if (!m_source_read_attempted) + if (m_owned_document_kind != OwnedDocumentKind::Source) { SceneDiagnosticError(diagnostics, "Source population requires an instance-owned source read.", SceneDiagnosticCode::InvalidArgument); return false; @@ -124,9 +126,72 @@ namespace ZEngine::ECS return PopulateDocument(m_document, schemas, diagnostics); } + bool SceneInstance::ReadCooked(const Managers::CookedObjectSnapshot& object, const SceneComponentSchemaRegistry& schemas, size_t working_bytes, SceneDiagnostics* diagnostics, const SceneDocumentLimits& limits) + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Worker.CookedDecode"); + if (!m_initialized || m_owned_document_kind != OwnedDocumentKind::None || m_population_attempted || m_document.Settings() || m_world.AliveCount() || !working_bytes || object.Bytes().empty()) + { + SceneDiagnosticError(diagnostics, "Cooked reconstruction requires a fresh owner, verified bytes and a finite document workspace.", SceneDiagnosticCode::InvalidArgument); + return false; + } + m_owned_document_kind = OwnedDocumentKind::Cooked; + if (!m_document_route.Initialize(Arena(), working_bytes)) + { + SceneDiagnosticError(diagnostics, "Cannot reserve the document workspace within admitted scene storage.", SceneDiagnosticCode::AllocationFailed); + return false; + } + const bool read = BinarySceneCodec::Decode(object.Bytes(), schemas, &m_document, diagnostics, limits, &m_document_route); + RecordMemoryUsage(); + return read; + } + + bool SceneInstance::ReadVerifiedCooked(const Managers::CookedPackageSnapshot& package, const Managers::CookedObjectSnapshot& object, const SceneComponentSchemaRegistry& schemas, size_t working_bytes, SceneDiagnostics* diagnostics, const SceneDocumentLimits& limits, const std::atomic* cancelled) + { + const auto* scene = package.Scene(); + if (!scene || object.Descriptor() != scene->Result.Object) + { + SceneDiagnosticError(diagnostics, "Cooked scene bytes do not match the selected package object.", SceneDiagnosticCode::InvalidData); + return false; + } + if (cancelled && cancelled->load(std::memory_order_acquire)) + { + SceneDiagnosticError(diagnostics, "Cooked scene reconstruction was cancelled.", SceneDiagnosticCode::InvalidData); + return false; + } + if (!ReadCooked(object, schemas, working_bytes, diagnostics, limits)) + { + return false; + } + if (package.ValidateScene(m_document, schemas, diagnostics, cancelled).Failed()) + { + SceneDiagnosticError(diagnostics, "Cooked scene references do not match the selected package.", SceneDiagnosticCode::InvalidData); + return false; + } + m_cooked_validated = true; + m_cooked_manifest = package.ManifestDigest(); + return true; + } + + bool SceneInstance::MatchesCookedClosure(const Core::Hash::Digest256& manifest, const uuids::uuid& scene) const noexcept + { + const auto* settings = m_document.Settings(); + return m_cooked_validated && settings && settings->UUID == scene && m_cooked_manifest == manifest; + } + + bool SceneInstance::PopulateCooked(const SceneComponentSchemaRegistry& schemas, SceneDiagnostics* diagnostics) + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Worker.Populate"); + if (m_owned_document_kind != OwnedDocumentKind::Cooked || !m_cooked_validated) + { + SceneDiagnosticError(diagnostics, "Cooked population requires a package-validated instance-owned decode.", SceneDiagnosticCode::InvalidArgument); + return false; + } + return PopulateDocument(m_document, schemas, diagnostics); + } + bool SceneInstance::TryReleaseDocumentForDiscard(SceneDocument* out) noexcept { - if (!out || m_source_read_attempted || !m_document.Settings()) + if (!out || m_owned_document_kind != OwnedDocumentKind::None || !m_document.Settings()) { return false; } diff --git a/ZEngine/ZEngine/ECS/SceneInstance.h b/ZEngine/ZEngine/ECS/SceneInstance.h index 5dccb2a9f..7c1ea9f29 100644 --- a/ZEngine/ZEngine/ECS/SceneInstance.h +++ b/ZEngine/ZEngine/ECS/SceneInstance.h @@ -7,8 +7,14 @@ #include #include #include +#include #include +namespace ZEngine::Managers +{ + class CookedPackageSnapshot; +} + namespace ZEngine::ECS { /// Stable-address engine owner, transferable through unique ownership. No @@ -36,6 +42,17 @@ namespace ZEngine::ECS [[nodiscard]] bool ReadSource(Core::VFS::IVFSContext& vfs, const Core::VFS::VFSPath& path, const SceneComponentSchemaRegistry& schemas, size_t working_bytes, SceneDiagnostics* diagnostics, const SceneDocumentLimits& limits = {}, const SceneMigrationRegistry* migrations = nullptr); [[nodiscard]] bool PopulateSource(const SceneComponentSchemaRegistry& schemas, SceneDiagnostics* diagnostics); + /// Decode an already verified private object snapshot into this owner's + /// bounded document route. This alone does not authorize population. + [[nodiscard]] bool ReadCooked(const Managers::CookedObjectSnapshot& object, const SceneComponentSchemaRegistry& schemas, size_t working_bytes, SceneDiagnostics* diagnostics, const SceneDocumentLimits& limits = {}); + /// Require the selected manifest object and its complete typed scene + /// references before allowing population of the unpublished world. + [[nodiscard]] bool ReadVerifiedCooked(const Managers::CookedPackageSnapshot& package, const Managers::CookedObjectSnapshot& object, const SceneComponentSchemaRegistry& schemas, size_t working_bytes, SceneDiagnostics* diagnostics, const SceneDocumentLimits& limits = {}, const std::atomic* cancelled = nullptr); + [[nodiscard]] bool PopulateCooked(const SceneComponentSchemaRegistry& schemas, SceneDiagnostics* diagnostics); + /// True only for the manifest and selected scene that authorized this + /// owner's cooked reconstruction. + [[nodiscard]] bool MatchesCookedClosure(const Core::Hash::Digest256& manifest, const uuids::uuid& scene) const noexcept; + /// Adapter rollback only: return the input snapshot before immediately /// discarding an unpublished instance whose later preparation failed. /// Internally routed documents cannot escape their storage owner. @@ -96,19 +113,27 @@ namespace ZEngine::ECS } private: + enum class OwnedDocumentKind : uint8_t + { + None, + Source, + Cooked, + }; SceneInstance() = default; [[nodiscard]] bool PopulateDocument(const SceneDocument& document, const SceneComponentSchemaRegistry& schemas, SceneDiagnostics* diagnostics); - SceneStorageLease m_storage = {}; - Managers::AssetCacheLease m_source_assets = {}; - Core::Memory::SlabAllocationRoute m_document_route = {}; - SceneDocument m_document = {}; - Scene m_world = {}; - ActorManager m_actors = {}; - WorldCommands m_commands = {}; - WorldTick m_tick = {}; - bool m_initialized = false; - bool m_population_attempted = false; - bool m_source_read_attempted = false; + SceneStorageLease m_storage = {}; + Managers::AssetCacheLease m_source_assets = {}; + Core::Memory::SlabAllocationRoute m_document_route = {}; + SceneDocument m_document = {}; + Scene m_world = {}; + ActorManager m_actors = {}; + WorldCommands m_commands = {}; + WorldTick m_tick = {}; + bool m_initialized = false; + bool m_population_attempted = false; + bool m_cooked_validated = false; + Core::Hash::Digest256 m_cooked_manifest = {}; + OwnedDocumentKind m_owned_document_kind = OwnedDocumentKind::None; }; } // namespace ZEngine::ECS diff --git a/ZEngine/ZEngine/ECS/SceneRuntimeCamera.cpp b/ZEngine/ZEngine/ECS/SceneRuntimeCamera.cpp new file mode 100644 index 000000000..765872f32 --- /dev/null +++ b/ZEngine/ZEngine/ECS/SceneRuntimeCamera.cpp @@ -0,0 +1,66 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace ZEngine::ECS +{ + bool CaptureSceneRuntimeCamera(const Scene& scene, const uuids::uuid& requested_camera, uint32_t viewport_width, uint32_t viewport_height, Rendering::Cameras::CameraFrameData* output, SceneDiagnostics* diagnostics) + { + if (!output || !viewport_width || !viewport_height) + { + SceneDiagnosticError(diagnostics, "A runtime camera requires an output and a nonzero viewport.", SceneDiagnosticCode::InvalidArgument); + return false; + } + const Components::CameraComponent* camera = nullptr; + const Components::TransformComponent* transform = nullptr; + EntityID selected = {}; + uint32_t matches = 0; + scene.ForEachEntity([&](EntityID entity) { + const auto* current = scene.GetComponent(entity); + const auto* id = scene.GetComponent(entity); + if (current && (requested_camera.is_nil() ? current->IsMain : id && id->Value == requested_camera)) + { + ++matches; + camera = current; + transform = scene.GetComponent(entity); + selected = entity; + } + }); + if (matches != 1 || !transform || !camera) + { + SceneDiagnosticError(diagnostics, "The requested runtime camera is missing, ambiguous, or has no transform.", SceneDiagnosticCode::InvalidData); + return false; + } + const auto* node = scene.Hierarchy().Find(selected); + const float aspect = static_cast(viewport_width) / static_cast(viewport_height); + if (!node || !node->TopologyValid || !node->FrameValid || !std::isfinite(camera->FovY) || camera->FovY <= 0.f || camera->FovY >= 180.f || !std::isfinite(camera->Near) || !std::isfinite(camera->Far) || camera->Near <= 0.f || camera->Far <= camera->Near || !std::isfinite(aspect) || aspect <= 0.f) + { + SceneDiagnosticError(diagnostics, "The runtime camera has an invalid hierarchy, transform, or projection.", SceneDiagnosticCode::InvalidData); + return false; + } + Rendering::Cameras::CameraFrameData result = {}; + if (!Core::Maths::Geometry::TryInverse(transform->WorldTransform, &result.View)) + { + SceneDiagnosticError(diagnostics, "The runtime camera transform is not invertible.", SceneDiagnosticCode::InvalidData); + return false; + } + const float tangent = std::tan(Core::Maths::radians(camera->FovY) * 0.5f); + const float n = camera->Near; + const float f = camera->Far; + result.Projection = Core::Maths::Mat4f(1.f / (aspect * tangent), 0.f, 0.f, 0.f, 0.f, -1.f / tangent, 0.f, 0.f, 0.f, 0.f, f / (n - f), (n * f) / (n - f), 0.f, 0.f, -1.f, 0.f); + result.Position = {transform->WorldTransform(0, 3), transform->WorldTransform(1, 3), transform->WorldTransform(2, 3)}; + result.ProjectionType = Rendering::Cameras::CameraType::PERSPECTIVE; + Core::Maths::Geometry::ProjectionFrame verified = {}; + if (!Core::Maths::Geometry::MakeProjectionFrame(result.View, result.Projection, result.Position, Core::Maths::Geometry::ProjectionKind::Perspective, false, &verified)) + { + SceneDiagnosticError(diagnostics, "The runtime camera cannot produce a finite projection.", SceneDiagnosticCode::InvalidData); + return false; + } + *output = result; + return true; + } +} // namespace ZEngine::ECS diff --git a/ZEngine/ZEngine/ECS/SceneRuntimeCamera.h b/ZEngine/ZEngine/ECS/SceneRuntimeCamera.h new file mode 100644 index 000000000..e9a606b1f --- /dev/null +++ b/ZEngine/ZEngine/ECS/SceneRuntimeCamera.h @@ -0,0 +1,13 @@ +#pragma once +#include +#include +#include +#include + +namespace ZEngine::ECS +{ + /// Capture a renderer-only launch view after candidate hierarchy sync. A + /// non-nil UUID selects that camera; otherwise exactly one main is required. + /// Failure preserves the output and never mutates the candidate scene. + [[nodiscard]] bool CaptureSceneRuntimeCamera(const Scene& scene, const uuids::uuid& requested_camera, uint32_t viewport_width, uint32_t viewport_height, Rendering::Cameras::CameraFrameData* output, SceneDiagnostics* diagnostics); +} // namespace ZEngine::ECS diff --git a/ZEngine/ZEngine/Engine.cpp b/ZEngine/ZEngine/Engine.cpp index 4ef83b879..70a7af6b2 100644 --- a/ZEngine/ZEngine/Engine.cpp +++ b/ZEngine/ZEngine/Engine.cpp @@ -156,6 +156,8 @@ namespace ZEngine auto vfs_ctx = ZPushStructCtor(&g_engine_ctx->VFSArena); vfs_ctx->Initialize(&g_engine_ctx->VFSArena); g_engine_ctx->VFS = vfs_ctx; + if (app->WantsSourceIndependenceAudit()) + vfs_ctx->EnableEngineOnlyAudit(); { g_engine_ctx->EngineAssetsBackend.Initialize(g_engine_ctx->EngineAssetsNativeRoot, Core::VFS::VFSBackendCaps::Read | Core::VFS::VFSBackendCaps::Write | Core::VFS::VFSBackendCaps::List, &g_engine_ctx->VFSArena); @@ -189,6 +191,8 @@ namespace ZEngine memory->CreateBudgetedArena(memory->Budget.AssetManager, &g_engine_ctx->AssetArena); Managers::AssetManager::Initialize(&g_engine_ctx->AssetArena, g_engine_ctx->Device, app->WorkingSpacePath); + if (app->WantsSourceIndependenceAudit()) + Managers::AssetManager::Instance()->Registry->EnableLookupDenialAudit(); memory->CreateBudgetedArena(memory->Budget.Input, &g_engine_ctx->InputArena); g_engine_ctx->InputManager = ZPushStructCtor(&arena); @@ -227,6 +231,8 @@ namespace ZEngine memory->CreateBudgetedArena(memory->Budget.EditorSceneLoadB, &g_engine_ctx->EditorSceneLoadArenaB); g_engine_ctx->ImportCoordinator = ZPushStructCtor(&g_engine_ctx->AssetArena); g_engine_ctx->ImportCoordinator->Initialize(&g_engine_ctx->AssetArena, g_engine_ctx->VFS, Managers::AssetManager::Instance()->Registry); + if (app->WantsSourceIndependenceAudit()) + g_engine_ctx->ImportCoordinator->EnableImportDenialAudit(); g_engine_ctx->GltfImporter.Initialize(&g_engine_ctx->ImportPipelineArena); g_engine_ctx->FbxImporter.Initialize(&g_engine_ctx->ImportPipelineArena); @@ -281,6 +287,7 @@ namespace ZEngine { // Step 1 — signal all loops to exit g_engine_ctx->SceneLoads.Shutdown(); + g_engine_ctx->App->RenderPipeline->CookedSceneUploads.Seal(); g_engine_ctx->RequestTerminate.value.store(true, std::memory_order_release); // Step 2 — join render thread before any GPU resource is destroyed @@ -294,6 +301,22 @@ namespace ZEngine // publish a completion into torn-down state. Helpers::ThreadPoolHelper::Shutdown(); + auto& cooked_uploads = g_engine_ctx->App->RenderPipeline->CookedSceneUploads; + if (!cooked_uploads.Drained()) + { + // The render owner has joined and producers can no longer publish. + // No recorded upload may be submitted after this idle/loss barrier. + const VkResult idle = vkDeviceWaitIdle(g_engine_ctx->Device->LogicalDevice); + if (idle == VK_ERROR_DEVICE_LOST) + { + g_engine_ctx->Device->CheckDeviceLost(idle, "Cooked scene shutdown"); + } + ZENGINE_VALIDATE_ASSERT(idle == VK_SUCCESS || idle == VK_ERROR_DEVICE_LOST, "Cannot retire cooked scene uploads without a quiescent device") + const bool drained = (idle == VK_SUCCESS || idle == VK_ERROR_DEVICE_LOST) && cooked_uploads.DrainAfterDeviceIdle(); + ZENGINE_VALIDATE_ASSERT(drained, "Cooked scene uploads survived joined-owner shutdown") + (void) drained; + } + Core::MainThreadScheduler::Shutdown(); g_engine_ctx->App->OnWorldShutdown(*g_engine_ctx); @@ -453,7 +476,10 @@ namespace ZEngine previous_main_start = main_work_start; // Platform events - window->PollEvent(); + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Main.PlatformEvents"); + window->PollEvent(); + } if (ProcessApplicationCloseRequest(*g_engine_ctx)) break; @@ -508,6 +534,7 @@ namespace ZEngine // input, simulation, or the state read by the render thread. if (g_engine_ctx->App->EnableRenderOverlay) { + ZENGINE_PROFILE_SCOPE("SceneLoad.Main.OverlayBuild"); uint32_t overlay_slot = 0; Applications::OverlayPayload* overlay = nullptr; if (pipeline->BeginOverlayWrite(overlay, overlay_slot)) @@ -522,19 +549,26 @@ namespace ZEngine if (g_engine_ctx->Scene && g_engine_ctx->App->CurrentScene) { + ZENGINE_PROFILE_SCOPE("SceneLoad.Main.SceneSynchronization"); ECS::Systems::SyncHierarchy(*g_engine_ctx->Scene); ECS::Systems::SyncECSToRenderScene(*g_engine_ctx->Scene, alpha, *g_engine_ctx->App->CurrentScene); ECS::Systems::SyncECSToLights(*g_engine_ctx->Scene, *g_engine_ctx->App->CurrentScene); } state.Gizmo.Reset(); - g_engine_ctx->App->PrepareScene(state); + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Main.PrepareScene"); + g_engine_ctx->App->PrepareScene(state); + } state.SceneGeneration = g_engine_ctx->ScenePublication.Generation(); state.FramebufferW = window->GetWidth(); state.FramebufferH = window->GetHeight(); state.RenderOverlay = g_engine_ctx->App->EnableRenderOverlay; // Transient mailbox pressure is expected; the consumer retains its valid frame. - pipeline->PublishFrameState(state, g_engine_ctx->App->CurrentScene); + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Main.PublishFrame"); + pipeline->PublishFrameState(state, g_engine_ctx->App->CurrentScene); + } ZENGINE_PROFILE_VALUE("SceneLoad.Main.FrameWorkMs", (std::chrono::duration(std::chrono::steady_clock::now() - main_work_start).count())); // Frame rate cap — applied unconditionally. @@ -616,7 +650,14 @@ namespace ZEngine }(); pipeline->Device->SwapchainPtr->UpdateFramebufferExtent(state.FramebufferW, state.FramebufferH); - const bool frame_valid = pipeline->BeginFrame(); + // Cancellation drains existing work without recording uploads. Service it before + // acquisition, which can wait on the presentation system or have no valid frame. + const bool cancellation_pumped = !pipeline->Device->IsDeviceLost.load(std::memory_order_acquire) && pipeline->CookedSceneUploads.PumpOne(0, pipeline->RenderMainThreadIndex, true); + const bool frame_valid = pipeline->BeginFrame(); + if (frame_valid && !cancellation_pumped) + { + (void) pipeline->CookedSceneUploads.PumpOne(pipeline->Device->SwapchainPtr->CurrentFrame->Index, pipeline->RenderMainThreadIndex); + } if (frame_valid && target_ready && state.Scene.HasScene) { const auto* overlay = state.RenderOverlay ? (has_next_overlay ? &next_overlay->ZUIOverlay : (retained_overlay ? &retained_overlay->ZUIOverlay : nullptr)) : nullptr; diff --git a/ZEngine/ZEngine/Hardwares/AsyncUploadQueue.h b/ZEngine/ZEngine/Hardwares/AsyncUploadQueue.h index c2713c6f3..f9b013448 100644 --- a/ZEngine/ZEngine/Hardwares/AsyncUploadQueue.h +++ b/ZEngine/ZEngine/Hardwares/AsyncUploadQueue.h @@ -66,6 +66,11 @@ namespace ZEngine::Hardwares { return !m_jobs.empty(); } + /// Producer-thread preflight; no other producer may enqueue before use. + bool HasCapacity() const + { + return !m_jobs.full(); + } private: VulkanDevice* m_device = nullptr; diff --git a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp index 6cdc5660f..74922998a 100644 --- a/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp +++ b/ZEngine/ZEngine/Hardwares/DeviceSwapchain.cpp @@ -3,12 +3,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include using namespace ZEngine::Core::Containers; using namespace ZEngine::Rendering; @@ -727,7 +729,30 @@ namespace ZEngine::Hardwares #endif const auto submit = [&]() { ZENGINE_PROFILE_SCOPE("SceneLoad.Render.QueueSubmit"); - return vkQueueSubmit2(queue.Handle, 1, &submit_info_1, CurrentFrame->Fence->GetHandle()); +#if ZENGINE_PROFILING + const auto wall_begin = std::chrono::steady_clock::now(); + const auto cpu_begin = Profiling::ReadThreadCpuTime(); +#endif + const auto result = vkQueueSubmit2(queue.Handle, 1, &submit_info_1, CurrentFrame->Fence->GetHandle()); +#if ZENGINE_PROFILING + const auto cpu_end = Profiling::ReadThreadCpuTime(); + const auto wall_ns = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - wall_begin).count()); + if (wall_ns >= 1'000'000) + { + const auto execution = Profiling::ThreadExecutionTimes::Between(cpu_begin, cpu_end, wall_ns); + ZENGINE_PROFILE_VALUE("SceneLoad.QueueSubmit.LongWallMs", wall_ns / 1'000'000.0); + if (execution.Valid) + { + ZENGINE_PROFILE_VALUE("SceneLoad.QueueSubmit.LongCpuMs", execution.CpuNanoseconds / 1'000'000.0); + ZENGINE_PROFILE_VALUE("SceneLoad.QueueSubmit.LongNonCpuMs", execution.NonCpuNanoseconds / 1'000'000.0); + } + else + { + ZENGINE_PROFILE_SCOPE("SceneLoad.QueueSubmit.CpuClockUnavailable"); + } + } +#endif + return result; }(); #ifndef NDEBUG if (TraceSubmission) diff --git a/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp b/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp index 28f172d97..26dc613e3 100644 --- a/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp +++ b/ZEngine/ZEngine/Hardwares/VulkanDevice.cpp @@ -112,6 +112,14 @@ namespace ZEngine::Hardwares } #endif + bool khronos_validation_enabled = false; + for (const char* layer_name : enabled_layer_name_collection) + { + if (Helpers::secure_strcmp(layer_name, "VK_LAYER_KHRONOS_validation") == 0) + khronos_validation_enabled = true; + } + ZENGINE_CORE_INFO("[VulkanValidation] khronos_enabled={}", khronos_validation_enabled ? 1 : 0) + Array enabled_extension_layer_name_collection; enabled_extension_layer_name_collection.init(scratch.Arena, 16); @@ -1199,9 +1207,11 @@ namespace ZEngine::Hardwares VkSemaphoreSubmitInfo signal_info = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = signal_semaphore->GetHandle(), .value = signal_value, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; VkSubmitInfo2 submit_info = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .waitSemaphoreInfoCount = wait_info_count, .pWaitSemaphoreInfos = wait_infos, .commandBufferInfoCount = 1, .pCommandBufferInfos = &command_info, .signalSemaphoreInfoCount = 1, .pSignalSemaphoreInfos = &signal_info}; VkResult submit_result = vkQueueSubmit2(GetQueue(command_buffer->QueueType).Handle, 1, &submit_info, VK_NULL_HANDLE); - if (CheckDeviceLost(submit_result, "QueueSubmit (timeline waits)")) + if (submit_result != VK_SUCCESS) + { + CheckDeviceLost(submit_result, "QueueSubmit (timeline waits)"); return false; - ZENGINE_VALIDATE_ASSERT(submit_result == VK_SUCCESS, "Failed to submit queue") + } command_buffer->SetState(CommandBufferState::Pending); return true; } @@ -2916,7 +2926,7 @@ namespace ZEngine::Hardwares info.usage = Specification.ImageUsage; info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; - const auto domain = (info.usage & (VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT)) ? Core::Memory::GpuMemoryDomain::RenderTarget : Core::Memory::GpuMemoryDomain::DeviceTexture; + const auto domain = Specification.IsCookedAsset ? Core::Memory::GpuMemoryDomain::DeviceTexture : (info.usage & (VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT)) ? Core::Memory::GpuMemoryDomain::RenderTarget : Core::Memory::GpuMemoryDomain::DeviceTexture; auto image = device->GpuMem.TryAllocateImage(info, domain, device->LogicalDevice, Specification.ImageAspectFlag, Specifications::ImageViewTypeMap[VALUE_FROM_SPEC_MAP(image_view_type)], Specification.LayerCount, DebugName, &result); if (status) { @@ -3159,6 +3169,7 @@ namespace ZEngine::Hardwares buffer_res->Specification = {.Width = spec.Width, .Height = spec.Height, .Depth = spec.Depth, .ImageViewTypeValue = spec.Is3D ? Specifications::ImageViewType::TYPE_3D : spec.IsCubemap ? Specifications::ImageViewType::TYPE_CUBE : Specifications::ImageViewType::TYPE_2D, .BufferUsageType = spec.Is3D ? Specifications::ImageBufferUsageType::SINGLE_3D_IMAGE : spec.IsCubemap ? Specifications::ImageBufferUsageType::CUBEMAP : Specifications::ImageBufferUsageType::SINGLE_2D_IMAGE, .ImageFormat = image_format, .ImageAspectFlag = VkImageAspectFlagBits(image_aspect), .MipLevelCount = spec.MipLevelCount, .LayerCount = spec.LayerCount}; buffer_res->Specification.ImageUsage = VkImageUsageFlagBits(image_usage_attachment | transfert_dst_bit | transfert_src_bit | sampled_bit | storage_bit); buffer_res->Specification.IsAliasable = spec.IsAliasable; + buffer_res->Specification.IsCookedAsset = spec.IsCookedAsset; } Rendering::Textures::TextureHandle VulkanDevice::CreateTexture(const Rendering::Specifications::TextureSpecification& spec, cstring debug_name, uint32_t reserved_slots) @@ -3475,7 +3486,13 @@ namespace ZEngine::Hardwares *out_ring_offset = std::numeric_limits::max(); } - if (!handle.Valid() || !(data) || !(command_buf) || (RRM && !static_cast(RRM)->BeginTextureVersionWrite(handle))) + if (!handle.Valid() || !data || !command_buf) + { + return {}; + } + const auto* resource = GlobalTextures.Access(handle); + const VkDeviceSize upload_size = resource && data_size == 0 ? resource->BufferSize : data_size; + if (!resource || !resource->ContainsBaseMip(upload_size) || (RRM && !static_cast(RRM)->BeginTextureVersionWrite(handle))) { return {}; } @@ -3501,14 +3518,15 @@ namespace ZEngine::Hardwares return {}; } - const VkDeviceSize upload_size = data_size == 0 ? resource->BufferSize : data_size; - if (upload_size == 0 || upload_size > resource->BufferSize) + const VkDeviceSize available = data_size == 0 ? resource->BufferSize : data_size; + VkDeviceSize upload_size = 0; + if (!resource->ContainsBaseMip(available, &upload_size)) { return {}; } uint32_t ring_offset = 0; - void* ring_ptr = use_staging_ring ? GpuMem.Ring.Allocate(static_cast(upload_size), 4, &ring_offset) : nullptr; + void* ring_ptr = use_staging_ring && upload_size <= std::numeric_limits::max() ? GpuMem.Ring.Allocate(static_cast(upload_size), 4, &ring_offset) : nullptr; if (ring_ptr) { diff --git a/ZEngine/ZEngine/Hardwares/VulkanDevice.h b/ZEngine/ZEngine/Hardwares/VulkanDevice.h index 7c0c31845..1bf7d0fc8 100644 --- a/ZEngine/ZEngine/Hardwares/VulkanDevice.h +++ b/ZEngine/ZEngine/Hardwares/VulkanDevice.h @@ -524,6 +524,8 @@ namespace ZEngine::Hardwares /// to reclaim and a later allocation can overwrite it before the GPU reads it. /// @param use_staging_ring Set false for independently submitted work whose /// completion is not represented by RenderTimeline. + /// @param data_size Available bytes, or zero when the caller guarantees + /// that the base mip is present. Only mip zero is copied. BufferView WriteTextureData(CommandBufferPtr command_buf, const Rendering::Textures::TextureHandle& handle, const void* data, uint32_t* out_ring_offset = nullptr, bool use_staging_ring = true, VkDeviceSize data_size = 0); Rendering::Renderers::RenderPasses::RenderPass* CreateRenderPass(Rendering::Specifications::RenderPassSpecification spec); @@ -533,8 +535,7 @@ namespace ZEngine::Hardwares friend struct ::RRMUploadBatchTestHelper; friend class Rendering::RenderResourceManager; - // Only the upload owner may bypass the public write admission after - // establishing its exclusive, submission-scoped texture hold. + // Internal recording helper after the public write admission. BufferView WriteTextureDataAfterAdmission(CommandBufferPtr command_buf, const Rendering::Textures::TextureHandle& handle, const void* data, uint32_t* out_ring_offset, bool use_staging_ring, VkDeviceSize data_size); static constexpr uint32_t MaxSceneDescriptorTextures = 8192; diff --git a/ZEngine/ZEngine/Importers/ImageCodec.cpp b/ZEngine/ZEngine/Importers/ImageCodec.cpp index 272346b81..e279d0737 100644 --- a/ZEngine/ZEngine/Importers/ImageCodec.cpp +++ b/ZEngine/ZEngine/Importers/ImageCodec.cpp @@ -6,6 +6,7 @@ #include #include #include +#include // One shared CPU implementation. Cooking installs its checked route before // probing/decoding, including decoder-internal allocations and reallocations. @@ -239,7 +240,7 @@ namespace ZEngine::Importers::ImageCodec } } // namespace - Core::VFS::VFSResult ValidateEncodedImage(const AssetCodec::AssetByteSource& source, IAllocationRoute& route, const EncodedImageLimits& limits, EncodedImageDescription* output) + static Core::VFS::VFSResult ProcessEncodedImage(const AssetCodec::AssetByteSource& source, IAllocationRoute& route, const EncodedImageLimits& limits, EncodedImageDescription* output, uint8_t** retained_pixels) { if (!source.Read || !output) { @@ -287,7 +288,7 @@ namespace ZEngine::Importers::ImageCodec { return VFS::VFSResult::Fail(input.Error != VFS::VFSError::OK ? input.Error : VFS::VFSError::Corrupted); } - if (static_cast(width) > limits.MaxDimension || static_cast(height) > limits.MaxDimension || uint64_t(width) * uint64_t(height) > limits.MaxDecodedBytes / (4 * (bits / 8))) + if (static_cast(width) > limits.MaxDimension || static_cast(height) > limits.MaxDimension || uint64_t(width) * uint64_t(height) > limits.MaxDecodedBytes / (4 * (bits / 8)) || uint64_t(width) * uint64_t(height) > std::numeric_limits::max() / 4) { return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); } @@ -307,15 +308,103 @@ namespace ZEngine::Importers::ImageCodec info.bits_per_channel = 8; void* pixels = DecodeImage(kind, decoder, &decoded_width, &decoded_height, &decoded_channels, info); const bool valid = pixels && decoded_width == width && decoded_height == height && decoded_channels == channels && info.bits_per_channel == static_cast(bits); - Free(pixels); if (input.Error != VFS::VFSError::OK || !valid) { + Free(pixels); return VFS::VFSResult::Fail(input.Error != VFS::VFSError::OK ? input.Error : VFS::VFSError::Corrupted); } + if (retained_pixels) + { + const size_t pixel_bytes = static_cast(width) * static_cast(height) * 4; + auto* rgba = static_cast(pixels); + if (bits == 16) + { + const auto* source16 = static_cast(pixels); + for (size_t i = 0; i < pixel_bytes; ++i) + { + rgba[i] = static_cast(source16[i] >> 8); + } + } + const size_t row_bytes = static_cast(width) * 4; + for (size_t y = 0; y < static_cast(height) / 2; ++y) + { + auto* top = rgba + y * row_bytes; + auto* bottom = rgba + (static_cast(height) - 1 - y) * row_bytes; + for (size_t x = 0; x < row_bytes; ++x) + { + std::swap(top[x], bottom[x]); + } + } + *retained_pixels = rgba; + } + else + { + Free(pixels); + } *output = {kind, static_cast(width), static_cast(height), static_cast(channels), bits}; return VFS::VFSResult::Ok(); } + DecodedImage::~DecodedImage() + { + Reset(); + } + + DecodedImage::DecodedImage(DecodedImage&& other) noexcept : m_route(std::exchange(other.m_route, nullptr)), m_pixels(std::exchange(other.m_pixels, nullptr)), m_size(std::exchange(other.m_size, 0)), m_description(std::exchange(other.m_description, {})) {} + + DecodedImage& DecodedImage::operator=(DecodedImage&& other) noexcept + { + if (this != &other) + { + Reset(); + m_route = std::exchange(other.m_route, nullptr); + m_pixels = std::exchange(other.m_pixels, nullptr); + m_size = std::exchange(other.m_size, 0); + m_description = std::exchange(other.m_description, {}); + } + return *this; + } + + void DecodedImage::Reset() noexcept + { + if (m_pixels) + { + m_route->Free(m_pixels); + } + m_route = nullptr; + m_pixels = nullptr; + m_size = 0; + m_description = {}; + } + + Core::Containers::ArrayView DecodedImage::Pixels() const noexcept + { + return {m_pixels, m_size}; + } + + Core::VFS::VFSResult ValidateEncodedImage(const AssetCodec::AssetByteSource& source, IAllocationRoute& route, const EncodedImageLimits& limits, EncodedImageDescription* output) + { + return ProcessEncodedImage(source, route, limits, output, nullptr); + } + + Core::VFS::VFSResult DecodeEncodedImage(const AssetCodec::AssetByteSource& source, IAllocationRoute& route, const EncodedImageLimits& limits, DecodedImage* output) + { + if (!output) + { + return VFS::VFSResult::Fail(VFS::VFSError::InvalidPath); + } + DecodedImage candidate = {}; + const auto result = ProcessEncodedImage(source, route, limits, &candidate.m_description, &candidate.m_pixels); + if (result.Failed()) + { + return result; + } + candidate.m_route = &route; + candidate.m_size = static_cast(candidate.m_description.Width) * candidate.m_description.Height * 4; + *output = std::move(candidate); + return VFS::VFSResult::Ok(); + } + AllocationScope::AllocationScope(IAllocationRoute& route) noexcept : m_previous(CurrentRoute), m_current(&route) { CurrentRoute = &route; diff --git a/ZEngine/ZEngine/Importers/ImageCodec.h b/ZEngine/ZEngine/Importers/ImageCodec.h index c87270f72..e1161f26c 100644 --- a/ZEngine/ZEngine/Importers/ImageCodec.h +++ b/ZEngine/ZEngine/Importers/ImageCodec.h @@ -52,6 +52,32 @@ namespace ZEngine::Importers::ImageCodec uint32_t BitsPerChannel = 0; }; + /// Route-owned, bottom-up RGBA8 pixels. The borrowed allocation route must + /// outlive this owner; move across threads only when the route permits it. + class DecodedImage final + { + public: + DecodedImage() = default; + ~DecodedImage(); + DecodedImage(const DecodedImage&) = delete; + DecodedImage& operator=(const DecodedImage&) = delete; + DecodedImage(DecodedImage&&) noexcept; + DecodedImage& operator=(DecodedImage&&) noexcept; + void Reset() noexcept; + [[nodiscard]] Core::Containers::ArrayView Pixels() const noexcept; + [[nodiscard]] const EncodedImageDescription& Description() const noexcept + { + return m_description; + } + + private: + friend Core::VFS::VFSResult DecodeEncodedImage(const AssetCodec::AssetByteSource&, IAllocationRoute&, const EncodedImageLimits&, DecodedImage*); + IAllocationRoute* m_route = nullptr; + uint8_t* m_pixels = nullptr; + size_t m_size = 0; + EncodedImageDescription m_description = {}; + }; + // Supported encoded LDR formats only; GIF validates its first frame, as the // runtime decoder does. Recognize format bytes without allocating decoder // state, then preflight dimensions/native RGBA @@ -68,6 +94,9 @@ namespace ZEngine::Importers::ImageCodec // reads only after dimension/native-storage preflight. It is included in // principal fixed accounting; larger reads bypass this buffer. [[nodiscard]] Core::VFS::VFSResult ValidateEncodedImage(const AssetCodec::AssetByteSource&, IAllocationRoute&, const EncodedImageLimits&, EncodedImageDescription*); + /// Decode the same immutable formats and bounds into route-owned bottom-up + /// RGBA8 pixels for runtime upload. Failure leaves the output unchanged. + [[nodiscard]] Core::VFS::VFSResult DecodeEncodedImage(const AssetCodec::AssetByteSource&, IAllocationRoute&, const EncodedImageLimits&, DecodedImage*); // Outside an explicit scope, preserve the existing worker-slab/system-heap // route. These also serve short-lived encoded buffers alongside stb pixels. diff --git a/ZEngine/ZEngine/Importers/ImportCoordinator.cpp b/ZEngine/ZEngine/Importers/ImportCoordinator.cpp index 57faf7918..1eb4cd6d1 100644 --- a/ZEngine/ZEngine/Importers/ImportCoordinator.cpp +++ b/ZEngine/ZEngine/Importers/ImportCoordinator.cpp @@ -33,6 +33,11 @@ namespace ZEngine::Importers uuids::uuid ImportCoordinator::Enqueue(Core::VFS::VFSPath path, ImportPriority priority, ImportCallback cb) { + if (m_deny_import.load(std::memory_order_acquire)) + { + m_denied_imports.fetch_add(1, std::memory_order_relaxed); + return {}; + } if (!m_vfs_ctx) return {}; diff --git a/ZEngine/ZEngine/Importers/ImportCoordinator.h b/ZEngine/ZEngine/Importers/ImportCoordinator.h index a0d71b00e..e34634873 100644 --- a/ZEngine/ZEngine/Importers/ImportCoordinator.h +++ b/ZEngine/ZEngine/Importers/ImportCoordinator.h @@ -22,6 +22,14 @@ namespace ZEngine::Importers class ImportCoordinator { public: + void EnableImportDenialAudit() noexcept + { + m_deny_import.store(true, std::memory_order_release); + } + [[nodiscard]] uint64_t DeniedImportCount() const noexcept + { + return m_denied_imports.load(std::memory_order_acquire); + } static constexpr uint32_t MAX_IMPORTERS = 16; static constexpr uint32_t JOBS_PER_TICK = 4; static constexpr uint32_t MAX_REQUEUE_COUNT = 3; @@ -54,6 +62,8 @@ namespace ZEngine::Importers } private: + std::atomic m_deny_import = false; + std::atomic m_denied_imports = 0; struct ImportTask { ImportCoordinator* Owner = nullptr; diff --git a/ZEngine/ZEngine/Logging/Logger.cpp b/ZEngine/ZEngine/Logging/Logger.cpp index 036686217..e8d2ba149 100644 --- a/ZEngine/ZEngine/Logging/Logger.cpp +++ b/ZEngine/ZEngine/Logging/Logger.cpp @@ -82,7 +82,7 @@ namespace ZEngine::Logging spdlog::flush_every(configuration.PeriodicFlush); spdlog::flush_on(spdlog::level::critical); - s_sink = std::make_shared((log_directory_path / configuration.LogFilename).string(), 1024 * 1024, 5, false); + s_sink = std::make_shared((log_directory_path / configuration.LogFilename).string(), configuration.FileMaxBytes, configuration.FileMaxRotations, false); s_logger_collection.push(std::make_shared(configuration.EngineLoggerName, s_sink, spdlog::thread_pool())); for (auto& logger : s_logger_collection) diff --git a/ZEngine/ZEngine/Logging/LoggerConfiguration.h b/ZEngine/ZEngine/Logging/LoggerConfiguration.h index 359b56c20..1732c361d 100644 --- a/ZEngine/ZEngine/Logging/LoggerConfiguration.h +++ b/ZEngine/ZEngine/Logging/LoggerConfiguration.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include namespace ZEngine::Logging @@ -12,5 +13,7 @@ namespace ZEngine::Logging std::string CrashLogDir = "CrashDumps"; std::chrono::milliseconds PeriodicFlush = std::chrono::milliseconds(1); size_t RingBufferSize = 1024; + size_t FileMaxBytes = 1024 * 1024; + size_t FileMaxRotations = 5; }; } // namespace ZEngine::Logging diff --git a/ZEngine/ZEngine/Managers/CookedAssetReader.cpp b/ZEngine/ZEngine/Managers/CookedAssetReader.cpp new file mode 100644 index 000000000..fa2051db4 --- /dev/null +++ b/ZEngine/ZEngine/Managers/CookedAssetReader.cpp @@ -0,0 +1,264 @@ +#include +#include +#include +#include +#include + +namespace ZEngine::Managers +{ + namespace VFS = Core::VFS; + namespace Codec = Importers::AssetCodec; + namespace Image = Importers::ImageCodec; + + namespace + { + const Cook::CookManifestAsset* Match(const CookedPackageSnapshot& package, const Cook::CookAssetIdentity& identity, const CookedObjectSnapshot& object, Cook::CookAssetKind kind) + { + const auto* asset = package.FindSelectedAsset(identity); + return asset && identity.Kind == kind && asset->Format && !object.Bytes().empty() && object.Descriptor() == asset->Result.Object ? asset : nullptr; + } + + bool MeshDependenciesMatch(const CookedPackageSnapshot& package, const Cook::CookAssetIdentity& identity, const Importers::AssetMesh& mesh) + { + const auto dependencies = package.Dependencies(identity); + size_t matched = 0; + for (size_t index = 0; index < mesh.SubMeshes.size(); ++index) + { + const auto& uuid = mesh.SubMeshes[index].MaterialUUID; + if (uuid.is_nil()) + { + continue; + } + if (dependencies.empty()) + { + return false; + } + char slot[48] = {}; + const int length = std::snprintf(slot, sizeof(slot), "submeshes/%zu/material", index); + if (length <= 0 || static_cast(length) >= sizeof(slot)) + { + return false; + } + const Core::Containers::StringView key(slot, static_cast(length)); + const auto dependency = std::lower_bound(dependencies.begin(), dependencies.end(), key, [](const Cook::CookDependency& left, Core::Containers::StringView right) { return Cook::CookDependencySlotBefore(left.Slot, right); }); + const Cook::CookAssetIdentity expected = {Cook::CookAssetKind::Material, uuid}; + if (dependency == dependencies.end() || dependency->Identity != expected || !(dependency->Slot == key) || !package.FindSelectedAsset(expected)) + { + return false; + } + ++matched; + } + return matched == dependencies.size(); + } + + bool MaterialDependenciesMatch(const CookedPackageSnapshot& package, const Cook::CookAssetIdentity& identity, const Codec::MaterialAssetDocument& material) + { + const auto dependencies = package.Dependencies(identity); + size_t cursor = 0; + for (size_t index = 0; index < material.Textures.size(); ++index) + { + const auto& texture = material.Textures[index]; + if (texture.UUID.is_nil()) + { + continue; + } + if (cursor >= dependencies.size()) + { + return false; + } + const auto& dependency = dependencies[cursor++]; + const Cook::CookAssetIdentity expected = {Cook::CookAssetKind::Texture, texture.UUID}; + const auto* child = package.FindSelectedAsset(expected); + if (dependency.Identity != expected || !(dependency.Slot == Codec::MaterialTextureSlotKey(static_cast(index))) || !child || !(Core::Containers::StringView(texture.Path) == child->Result.Object.Path.View())) + { + return false; + } + } + return cursor == dependencies.size(); + } + + cstring TextureFormat(Image::EncodedImageKind kind) + { + switch (kind) + { + case Image::EncodedImageKind::PNG: + return "image-png"; + case Image::EncodedImageKind::JPEG: + return "image-jpeg"; + case Image::EncodedImageKind::BMP: + return "image-bmp"; + case Image::EncodedImageKind::TGA: + return "image-tga"; + case Image::EncodedImageKind::GIF: + return "image-gif"; + case Image::EncodedImageKind::PSD: + return "image-psd"; + case Image::EncodedImageKind::PIC: + return "image-pic"; + default: + return nullptr; + } + } + + const Cook::CookManifestAsset* MatchTexture(const CookedPackageSnapshot& package, const Cook::CookAssetIdentity& identity, const CookedObjectSnapshot& object) + { + const auto* asset = Match(package, identity, object, Cook::CookAssetKind::Texture); + return asset && package.Dependencies(identity).empty() ? asset : nullptr; + } + + bool MatchesTextureFormat(const Cook::CookManifestAsset& asset, Image::EncodedImageKind kind) + { + const auto format = TextureFormat(kind); + return format && Core::Containers::StringView(asset.Format->Key) == format; + } + } // namespace + + bool CookedAssetReader::MatchesMeshDependencies(const CookedPackageSnapshot& package, const Cook::CookAssetIdentity& identity, const Importers::AssetMesh& mesh) + { + const auto* asset = package.FindSelectedAsset(identity); + return asset && asset->Format && identity.Kind == Cook::CookAssetKind::Mesh && mesh.MeshUUID == identity.UUID && Core::Containers::StringView(asset->Format->Key) == "zemesh-v1-le64" && MeshDependenciesMatch(package, identity, mesh); + } + + bool CookedAssetReader::MatchesMaterialDependencies(const CookedPackageSnapshot& package, const Cook::CookAssetIdentity& identity, const Codec::MaterialAssetDocument& material) + { + const auto* asset = package.FindSelectedAsset(identity); + return asset && asset->Format && identity.Kind == Cook::CookAssetKind::Material && material.UUID == identity.UUID && Core::Containers::StringView(asset->Format->Key) == "zematerial-v1-json" && MaterialDependenciesMatch(package, identity, material); + } + + VFS::VFSResult CookedAssetReader::ReadMesh(const CookedPackageSnapshot& package, const Cook::CookAssetIdentity& identity, const CookedObjectSnapshot& object, Core::Memory::ArenaAllocator* arena, Core::Memory::IWorkingMemoryBudget& budget, Importers::AssetMesh* output, Importers::AssetNodeHierarchy* hierarchy, const Codec::MeshAssetReadLimits& limits) + { + if (!arena || !output || !hierarchy) + { + return VFS::VFSResult::Fail(VFS::VFSError::InvalidPath); + } + const auto* asset = Match(package, identity, object, Cook::CookAssetKind::Mesh); + if (!asset || !(Core::Containers::StringView(asset->Format->Key) == "zemesh-v1-le64")) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + Core::Memory::WorkingMemoryCharge fixed = {}; + if (!fixed.TryAcquire(&budget, Codec::MeshAssetReadFixedWorkingBytes() + sizeof(Importers::AssetMesh) + sizeof(Importers::AssetNodeHierarchy) + sizeof(fixed))) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + const auto temporary = Core::Memory::BeginTempArena(arena); + VFS::VFSError error = VFS::VFSError::OK; + { + Importers::AssetMesh candidate = {}; + Importers::AssetNodeHierarchy candidate_hierarchy = {}; + const auto decoded = Codec::ReadMeshAsset(object.Source(), arena, &candidate, &candidate_hierarchy, identity.UUID, limits); + if (decoded.Failed()) + { + error = decoded.Error(); + } + else if (!MatchesMeshDependencies(package, identity, candidate)) + { + error = VFS::VFSError::Corrupted; + } + else + { + *output = std::move(candidate); + *hierarchy = std::move(candidate_hierarchy); + } + } + if (error != VFS::VFSError::OK) + { + Core::Memory::EndTempArena(temporary); + return VFS::VFSResult::Fail(error); + } + return VFS::VFSResult::Ok(); + } + + VFS::VFSResult CookedAssetReader::ReadMaterial(const CookedPackageSnapshot& package, const Cook::CookAssetIdentity& identity, const CookedObjectSnapshot& object, Codec::MaterialAssetReader& reader, const Codec::MaterialAssetDocument** output) + { + if (!output) + { + return VFS::VFSResult::Fail(VFS::VFSError::InvalidPath); + } + const auto* asset = Match(package, identity, object, Cook::CookAssetKind::Material); + if (!asset || !(Core::Containers::StringView(asset->Format->Key) == "zematerial-v1-json")) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + const auto bytes = object.Bytes(); + const auto decoded = reader.Read({bytes.data(), bytes.size()}, identity.UUID); + if (decoded.Failed()) + { + return VFS::VFSResult::Fail(decoded.Error()); + } + if (!decoded.Value() || !MatchesMaterialDependencies(package, identity, *decoded.Value())) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + *output = decoded.Value(); + return VFS::VFSResult::Ok(); + } + + VFS::VFSResult CookedAssetReader::ValidateTexture(const CookedPackageSnapshot& package, const Cook::CookAssetIdentity& identity, const CookedObjectSnapshot& object, Core::Memory::IAllocationRoute& route, const Image::EncodedImageLimits& limits, Image::EncodedImageDescription* output) + { + if (!output) + { + return VFS::VFSResult::Fail(VFS::VFSError::InvalidPath); + } + const auto* asset = MatchTexture(package, identity, object); + if (!asset) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + Image::EncodedImageDescription description = {}; + const auto validated = Image::ValidateEncodedImage(object.Source(), route, limits, &description); + if (validated.Failed()) + { + return validated; + } + if (!MatchesTextureFormat(*asset, description.Kind)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + *output = description; + return VFS::VFSResult::Ok(); + } + + VFS::VFSResult CookedAssetReader::ReadTexture(const CookedPackageSnapshot& package, const Cook::CookAssetIdentity& identity, const CookedObjectSnapshot& object, Core::Memory::IAllocationRoute& route, const Image::EncodedImageLimits& limits, Image::DecodedImage* output) + { + if (!output) + { + return VFS::VFSResult::Fail(VFS::VFSError::InvalidPath); + } + const auto* asset = MatchTexture(package, identity, object); + if (!asset) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + Image::DecodedImage image = {}; + const auto decoded = Image::DecodeEncodedImage(object.Source(), route, limits, &image); + if (decoded.Failed()) + { + return decoded; + } + if (!MatchesTextureFormat(*asset, image.Description().Kind)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + *output = std::move(image); + return VFS::VFSResult::Ok(); + } + + VFS::VFSResult CookedAssetReader::ReadEnvironment(const CookedPackageSnapshot& package, const Cook::CookAssetIdentity& identity, const CookedObjectSnapshot& object, Core::Containers::ArrayView stream, Codec::EnvironmentMapFileHeader* output, const Codec::EnvironmentMapReadLimits& limits) + { + if (!output) + { + return VFS::VFSResult::Fail(VFS::VFSError::InvalidPath); + } + const auto* asset = Match(package, identity, object, Cook::CookAssetKind::EnvironmentMap); + if (!asset || !(Core::Containers::StringView(asset->Format->Key) == "zenvmap-v2-le64") || !package.Dependencies(identity).empty() || object.Bytes().size() < sizeof(Codec::EnvironmentMapFileHeader)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + Codec::EnvironmentMapFileHeader header = {}; + (void) Helpers::secure_memcpy(&header, sizeof(header), object.Bytes().data(), sizeof(header)); + // The immutable object digest, not the source-file receipt, is the + // runtime identity. The header's source hash is provenance only. + return Codec::ReadEnvironmentMapAsset(object.Source(), stream, output, header.SourceHash, limits); + } +} // namespace ZEngine::Managers diff --git a/ZEngine/ZEngine/Managers/CookedAssetReader.h b/ZEngine/ZEngine/Managers/CookedAssetReader.h new file mode 100644 index 000000000..a533b3620 --- /dev/null +++ b/ZEngine/ZEngine/Managers/CookedAssetReader.h @@ -0,0 +1,25 @@ +#pragma once +#include +#include +#include +#include + +namespace ZEngine::Managers +{ + /// Strict CPU decoders for verified package objects. These do not publish + /// cache entries, create GPU resources, or establish first-frame readiness. + /// All output storage/arenas and reader state are caller owned and bounded. + class CookedAssetReader final + { + public: + /// Revalidate package-specific edges when an already verified immutable + /// CPU version is reused by another selected scene closure. + [[nodiscard]] static bool MatchesMeshDependencies(const CookedPackageSnapshot&, const Cook::CookAssetIdentity&, const Importers::AssetMesh&); + [[nodiscard]] static bool MatchesMaterialDependencies(const CookedPackageSnapshot&, const Cook::CookAssetIdentity&, const Importers::AssetCodec::MaterialAssetDocument&); + [[nodiscard]] static Core::VFS::VFSResult ReadMesh(const CookedPackageSnapshot&, const Cook::CookAssetIdentity&, const CookedObjectSnapshot&, Core::Memory::ArenaAllocator*, Core::Memory::IWorkingMemoryBudget&, Importers::AssetMesh*, Importers::AssetNodeHierarchy*, const Importers::AssetCodec::MeshAssetReadLimits& = {}); + [[nodiscard]] static Core::VFS::VFSResult ReadMaterial(const CookedPackageSnapshot&, const Cook::CookAssetIdentity&, const CookedObjectSnapshot&, Importers::AssetCodec::MaterialAssetReader&, const Importers::AssetCodec::MaterialAssetDocument**); + [[nodiscard]] static Core::VFS::VFSResult ValidateTexture(const CookedPackageSnapshot&, const Cook::CookAssetIdentity&, const CookedObjectSnapshot&, Core::Memory::IAllocationRoute&, const Importers::ImageCodec::EncodedImageLimits&, Importers::ImageCodec::EncodedImageDescription*); + [[nodiscard]] static Core::VFS::VFSResult ReadTexture(const CookedPackageSnapshot&, const Cook::CookAssetIdentity&, const CookedObjectSnapshot&, Core::Memory::IAllocationRoute&, const Importers::ImageCodec::EncodedImageLimits&, Importers::ImageCodec::DecodedImage*); + [[nodiscard]] static Core::VFS::VFSResult ReadEnvironment(const CookedPackageSnapshot&, const Cook::CookAssetIdentity&, const CookedObjectSnapshot&, Core::Containers::ArrayView stream, Importers::AssetCodec::EnvironmentMapFileHeader*, const Importers::AssetCodec::EnvironmentMapReadLimits& = {}); + }; +} // namespace ZEngine::Managers diff --git a/ZEngine/ZEngine/Managers/CookedCpuAssetPool.h b/ZEngine/ZEngine/Managers/CookedCpuAssetPool.h new file mode 100644 index 000000000..4fb20d52f --- /dev/null +++ b/ZEngine/ZEngine/Managers/CookedCpuAssetPool.h @@ -0,0 +1,39 @@ +#pragma once +#include + +namespace ZEngine::Managers +{ + /// Worker-confined implementation detail of the bounded closure cache. + class CookedCpuAssetPool final + { + public: + CookedCpuAssetPool() = default; + ~CookedCpuAssetPool(); + CookedCpuAssetPool(const CookedCpuAssetPool&) = delete; + CookedCpuAssetPool& operator=(const CookedCpuAssetPool&) = delete; + + private: + friend class CookedCpuAssetSet; + friend class CookedCpuClosureCache; + struct Entry; + static constexpr uint32_t Invalid = UINT32_MAX; + [[nodiscard]] bool Initialize(Core::Memory::IAllocationRoute&, uint32_t capacity); + [[nodiscard]] bool Shutdown() noexcept; + [[nodiscard]] uint32_t Find(const CookedCpuAssetVersion&) const noexcept; + [[nodiscard]] bool Retain(uint32_t) noexcept; + [[nodiscard]] bool Publish(const CookedCpuAssetVersion&, void*, uint32_t*) noexcept; + void Release(uint32_t) noexcept; + [[nodiscard]] bool HasCapacity() const noexcept + { + return m_free != Invalid; + } + [[nodiscard]] uint32_t Bucket(const CookedCpuAssetVersion&) const noexcept; + Core::Memory::IAllocationRoute* m_route = nullptr; + Entry* m_entries = nullptr; + uint32_t* m_buckets = nullptr; + uint32_t m_capacity = 0; + uint32_t m_bucket_count = 0; + uint32_t m_free = Invalid; + uint32_t m_live = 0; + }; +} // namespace ZEngine::Managers diff --git a/ZEngine/ZEngine/Managers/CookedCpuAssetSet.cpp b/ZEngine/ZEngine/Managers/CookedCpuAssetSet.cpp new file mode 100644 index 000000000..30b2b15d1 --- /dev/null +++ b/ZEngine/ZEngine/Managers/CookedCpuAssetSet.cpp @@ -0,0 +1,640 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ZEngine::Managers +{ + namespace VFS = Core::VFS; + namespace Codec = Importers::AssetCodec; + namespace Image = Importers::ImageCodec; + + namespace + { + struct MeshPayload + { + void* Storage = nullptr; + Core::Memory::ArenaAllocator Arena = {}; + Importers::AssetMesh Mesh = {}; + Importers::AssetNodeHierarchy Hierarchy = {}; + }; + + struct MaterialPayload + { + Codec::MaterialAssetReader Reader = {}; + const Codec::MaterialAssetDocument* Document = nullptr; + }; + + struct TexturePayload + { + Image::DecodedImage Image = {}; + }; + + struct EnvironmentPayload + { + CookedObjectSnapshot Object = {}; + Codec::EnvironmentMapFileHeader Header = {}; + }; + + bool IsCancelled(const std::atomic* cancelled) + { + return cancelled && cancelled->load(std::memory_order_acquire); + } + + template + T* AllocatePayload(Core::Memory::IAllocationRoute& route) + { + static_assert(alignof(T) <= alignof(std::max_align_t)); + void* storage = route.Allocate(sizeof(T)); + return storage ? new (storage) T{} : nullptr; + } + + void DestroyPayload(Core::Memory::IAllocationRoute& route, Cook::CookAssetKind kind, void* payload) + { + if (!payload) + { + return; + } + switch (kind) + { + case Cook::CookAssetKind::Mesh: + { + auto* mesh = static_cast(payload); + void* storage = mesh->Storage; + mesh->~MeshPayload(); + if (storage) + { + route.Free(storage); + } + break; + } + case Cook::CookAssetKind::Material: + static_cast(payload)->~MaterialPayload(); + break; + case Cook::CookAssetKind::Texture: + static_cast(payload)->~TexturePayload(); + break; + case Cook::CookAssetKind::EnvironmentMap: + static_cast(payload)->~EnvironmentPayload(); + break; + default: + ZENGINE_VALIDATE_ASSERT(false, "Invalid cooked CPU payload kind") + break; + } + route.Free(payload); + } + } // namespace + + struct CookedCpuAssetPool::Entry + { + CookedCpuAssetVersion Version = {}; + void* Payload = nullptr; + uint32_t References = 0; + uint32_t Next = Invalid; + }; + + CookedCpuAssetPool::~CookedCpuAssetPool() + { + const bool closed = Shutdown(); + ZENGINE_VALIDATE_ASSERT(closed, "Cooked CPU asset pool destroyed while versions are retained.") + (void) closed; + } + + bool CookedCpuAssetPool::Initialize(Core::Memory::IAllocationRoute& route, uint32_t capacity) + { + if (m_route || !route.WorkingMemoryBudget() || capacity > UINT32_MAX / 2 || size_t(capacity) > std::numeric_limits::max() / sizeof(Entry)) + { + return false; + } + if (!capacity) + { + m_route = &route; + return true; + } + uint32_t bucket_count = 1; + while (bucket_count < capacity * 2) + { + if (bucket_count > UINT32_MAX / 2) + { + return false; + } + bucket_count *= 2; + } + if (size_t(bucket_count) > std::numeric_limits::max() / sizeof(uint32_t)) + { + return false; + } + static_assert(alignof(Entry) <= alignof(std::max_align_t)); + auto* entries = static_cast(route.Allocate(size_t(capacity) * sizeof(Entry))); + if (!entries) + { + return false; + } + auto* buckets = static_cast(route.Allocate(size_t(bucket_count) * sizeof(uint32_t))); + if (!buckets) + { + route.Free(entries); + return false; + } + for (uint32_t i = 0; i < capacity; ++i) + { + new (entries + i) Entry{}; + entries[i].Next = i + 1 < capacity ? i + 1 : Invalid; + } + std::fill(buckets, buckets + bucket_count, Invalid); + m_route = &route; + m_entries = entries; + m_buckets = buckets; + m_capacity = capacity; + m_bucket_count = bucket_count; + m_free = 0; + return true; + } + + bool CookedCpuAssetPool::Shutdown() noexcept + { + if (m_live) + { + return false; + } + if (m_entries) + { + for (uint32_t i = 0; i < m_capacity; ++i) + { + m_entries[i].~Entry(); + } + m_route->Free(m_entries); + m_route->Free(m_buckets); + } + m_route = nullptr; + m_entries = nullptr; + m_buckets = nullptr; + m_capacity = 0; + m_bucket_count = 0; + m_free = Invalid; + return true; + } + + uint32_t CookedCpuAssetPool::Bucket(const CookedCpuAssetVersion& version) const noexcept + { + const auto uuid = version.Identity.UUID.as_bytes(); + const auto seed = rapidhash_withSeed(uuid.data(), uuid.size(), static_cast(version.Identity.Kind)); + return static_cast(rapidhash_withSeed(version.ObjectDigest.Bytes.data(), version.ObjectDigest.Bytes.size(), seed)) & (m_bucket_count - 1); + } + + uint32_t CookedCpuAssetPool::Find(const CookedCpuAssetVersion& version) const noexcept + { + if (!m_bucket_count) + { + return Invalid; + } + for (uint32_t index = m_buckets[Bucket(version)]; index != Invalid; index = m_entries[index].Next) + { + const auto& existing = m_entries[index].Version; + if (existing.Identity == version.Identity && existing.ObjectDigest == version.ObjectDigest && existing.Format == version.Format && existing.ObjectBytes == version.ObjectBytes) + { + return index; + } + } + return Invalid; + } + + bool CookedCpuAssetPool::Retain(uint32_t index) noexcept + { + if (index >= m_capacity || !m_entries[index].References || m_entries[index].References == UINT32_MAX) + { + return false; + } + ++m_entries[index].References; + return true; + } + + bool CookedCpuAssetPool::Publish(const CookedCpuAssetVersion& version, void* payload, uint32_t* output) noexcept + { + if (!payload || !output || !m_bucket_count || m_free == Invalid || Find(version) != Invalid) + { + return false; + } + const uint32_t index = m_free; + auto& entry = m_entries[index]; + m_free = entry.Next; + const uint32_t bucket = Bucket(version); + entry.Version = version; + entry.Payload = payload; + entry.References = 1; + entry.Next = m_buckets[bucket]; + m_buckets[bucket] = index; + ++m_live; + *output = index; + return true; + } + + void CookedCpuAssetPool::Release(uint32_t index) noexcept + { + if (index >= m_capacity || !m_entries[index].References) + { + ZENGINE_VALIDATE_ASSERT(false, "Invalid cooked CPU version release") + return; + } + auto& entry = m_entries[index]; + if (--entry.References) + { + return; + } + auto* link = &m_buckets[Bucket(entry.Version)]; + while (*link != Invalid && *link != index) + { + link = &m_entries[*link].Next; + } + if (*link != index) + { + ZENGINE_VALIDATE_ASSERT(false, "Cooked CPU version is missing from its hash bucket") + return; + } + *link = entry.Next; + DestroyPayload(*m_route, entry.Version.Identity.Kind, entry.Payload); + entry = {}; + entry.Next = m_free; + m_free = index; + --m_live; + } + + struct CookedCpuAssetSet::Record + { + CookedCpuAssetVersion Version = {}; + void* Payload = nullptr; + uint32_t SharedIndex = CookedCpuAssetPool::Invalid; + }; + + struct CookedCpuAssetSet::BuildContext + { + CookedCpuAssetSet* Set = nullptr; + CookedPackageSnapshot* Package = nullptr; + const CookedCpuAssetLimits* Limits = nullptr; + const std::atomic* Cancelled = nullptr; + }; + + CookedCpuAssetSet::~CookedCpuAssetSet() + { + Reset(); + } + + CookedCpuAssetSet::CookedCpuAssetSet(CookedCpuAssetSet&& other) noexcept : m_route(std::exchange(other.m_route, nullptr)), m_pool(std::exchange(other.m_pool, nullptr)), m_records(std::exchange(other.m_records, nullptr)), m_count(std::exchange(other.m_count, 0)), m_capacity(std::exchange(other.m_capacity, 0)), m_manifest_digest(std::exchange(other.m_manifest_digest, {})), m_scene_uuid(std::exchange(other.m_scene_uuid, {})), m_valid(std::exchange(other.m_valid, false)) {} + + CookedCpuAssetSet& CookedCpuAssetSet::operator=(CookedCpuAssetSet&& other) noexcept + { + if (this != &other) + { + Reset(); + m_route = std::exchange(other.m_route, nullptr); + m_pool = std::exchange(other.m_pool, nullptr); + m_records = std::exchange(other.m_records, nullptr); + m_count = std::exchange(other.m_count, 0); + m_capacity = std::exchange(other.m_capacity, 0); + m_manifest_digest = std::exchange(other.m_manifest_digest, {}); + m_scene_uuid = std::exchange(other.m_scene_uuid, {}); + m_valid = std::exchange(other.m_valid, false); + } + return *this; + } + + void CookedCpuAssetSet::Reset() noexcept + { + for (size_t i = m_count; i > 0; --i) + { + auto& record = m_records[i - 1]; + if (record.Payload) + { + if (record.SharedIndex != CookedCpuAssetPool::Invalid) + { + m_pool->Release(record.SharedIndex); + } + else + { + DestroyPayload(*m_route, record.Version.Identity.Kind, record.Payload); + } + } + record.~Record(); + } + if (m_records) + { + m_route->Free(m_records); + } + m_route = nullptr; + m_pool = nullptr; + m_records = nullptr; + m_count = 0; + m_capacity = 0; + m_manifest_digest = {}; + m_scene_uuid = {}; + m_valid = false; + } + + VFS::VFSResult CookedCpuAssetSet::BuildAsset(void* context, const Cook::CookManifestAsset& asset) + { + auto& build = *static_cast(context); + auto& set = *build.Set; + auto& route = *set.m_route; + if (IsCancelled(build.Cancelled)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Cancelled); + } + if (!asset.Format || set.m_count == set.m_capacity) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + auto& record = *new (set.m_records + set.m_count++) Record{}; + record.Version = {asset.Identity, asset.Result.Object.Digest, asset.Result.InputKey, asset.Format, asset.Result.Object.Size}; + if (set.m_pool) + { + const uint32_t index = set.m_pool->Find(record.Version); + if (index != CookedCpuAssetPool::Invalid) + { + void* payload = set.m_pool->m_entries[index].Payload; + bool matches = false; + switch (asset.Identity.Kind) + { + case Cook::CookAssetKind::Mesh: + matches = CookedAssetReader::MatchesMeshDependencies(*build.Package, asset.Identity, static_cast(payload)->Mesh); + break; + case Cook::CookAssetKind::Material: + { + const auto* material = static_cast(payload)->Document; + matches = material && CookedAssetReader::MatchesMaterialDependencies(*build.Package, asset.Identity, *material); + break; + } + case Cook::CookAssetKind::Texture: + case Cook::CookAssetKind::EnvironmentMap: + matches = build.Package->Dependencies(asset.Identity).empty(); + break; + default: + break; + } + if (!matches) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + if (!set.m_pool->Retain(index)) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + record.Payload = payload; + record.SharedIndex = index; + return IsCancelled(build.Cancelled) ? VFS::VFSResult::Fail(VFS::VFSError::Cancelled) : VFS::VFSResult::Ok(); + } + if (!set.m_pool->HasCapacity()) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + } + CookedObjectSnapshot object = {}; + const auto captured = build.Package->CaptureAsset(asset.Identity, route, build.Limits->MaxObjectBytes, &object, build.Cancelled); + if (captured.Failed()) + { + return captured; + } + VFS::VFSResult result = VFS::VFSResult::Fail(VFS::VFSError::Unsupported); + switch (asset.Identity.Kind) + { + case Cook::CookAssetKind::Mesh: + { + size_t bytes = 0; + { + Core::Memory::WorkingMemoryCharge fixed = {}; + if (!fixed.TryAcquire(route.WorkingMemoryBudget(), Codec::MeshAssetReadFixedWorkingBytes() + sizeof(fixed))) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + result = Codec::MeasureMeshAssetReadWorkspace(object.Source(), &bytes, asset.Identity.UUID, build.Limits->Mesh); + } + if (result.Failed()) + { + return result; + } + if (!bytes || bytes > build.Limits->MaxMeshWorkspaceBytes) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + if (bytes > std::numeric_limits::max() - (DEFAULT_ALIGNMENT - 1)) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + auto* mesh = AllocatePayload(route); + if (!mesh) + { + return VFS::VFSResult::Fail(VFS::VFSError::OutOfMemory); + } + record.Payload = mesh; + size_t storage_bytes = bytes + DEFAULT_ALIGNMENT - 1; + mesh->Storage = route.Allocate(storage_bytes); + if (!mesh->Storage) + { + return VFS::VFSResult::Fail(VFS::VFSError::OutOfMemory); + } + // A route guarantees max_align_t, which can be less than the + // alignment assumed by the mesh workspace measurement. + void* aligned = mesh->Storage; + if (!std::align(DEFAULT_ALIGNMENT, bytes, aligned, storage_bytes) || !mesh->Arena.InitializeBorrowed(aligned, bytes, "CookedCpuMesh")) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + result = CookedAssetReader::ReadMesh(*build.Package, asset.Identity, object, &mesh->Arena, *route.WorkingMemoryBudget(), &mesh->Mesh, &mesh->Hierarchy, build.Limits->Mesh); + break; + } + case Cook::CookAssetKind::Material: + { + auto* material = AllocatePayload(route); + if (!material) + { + return VFS::VFSResult::Fail(VFS::VFSError::OutOfMemory); + } + record.Payload = material; + result = CookedAssetReader::ReadMaterial(*build.Package, asset.Identity, object, material->Reader, &material->Document); + break; + } + case Cook::CookAssetKind::Texture: + { + auto* texture = AllocatePayload(route); + if (!texture) + { + return VFS::VFSResult::Fail(VFS::VFSError::OutOfMemory); + } + record.Payload = texture; + result = CookedAssetReader::ReadTexture(*build.Package, asset.Identity, object, route, build.Limits->Texture, &texture->Image); + break; + } + case Cook::CookAssetKind::EnvironmentMap: + { + auto* environment = AllocatePayload(route); + if (!environment) + { + return VFS::VFSResult::Fail(VFS::VFSError::OutOfMemory); + } + record.Payload = environment; + Core::Memory::WorkingMemoryCharge fixed = {}; + if (!fixed.TryAcquire(route.WorkingMemoryBudget(), ZKilo(4) + sizeof(fixed))) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + uint8_t stream[ZKilo(4)] = {}; + result = CookedAssetReader::ReadEnvironment(*build.Package, asset.Identity, object, stream, &environment->Header, build.Limits->Environment); + if (result.Succeeded()) + { + environment->Object = std::move(object); + } + break; + } + default: + break; + } + if (result.Failed()) + { + return result; + } + if (IsCancelled(build.Cancelled)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Cancelled); + } + if (set.m_pool && !set.m_pool->Publish(record.Version, record.Payload, &record.SharedIndex)) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + return VFS::VFSResult::Ok(); + } + + VFS::VFSResult CookedCpuAssetSet::Capture(CookedPackageSnapshot& package, Core::Memory::IAllocationRoute& route, const CookedCpuAssetLimits& limits, CookedCpuAssetSet* output, const std::atomic* cancelled) + { + return CaptureImpl(package, route, nullptr, limits, output, cancelled); + } + + VFS::VFSResult CookedCpuAssetSet::CaptureShared(CookedPackageSnapshot& package, Core::Memory::IAllocationRoute& route, CookedCpuAssetPool& pool, const CookedCpuAssetLimits& limits, CookedCpuAssetSet* output, const std::atomic* cancelled) + { + if (pool.m_route != &route) + { + return VFS::VFSResult::Fail(VFS::VFSError::InvalidPath); + } + return CaptureImpl(package, route, &pool, limits, output, cancelled); + } + + VFS::VFSResult CookedCpuAssetSet::CaptureImpl(CookedPackageSnapshot& package, Core::Memory::IAllocationRoute& route, CookedCpuAssetPool* pool, const CookedCpuAssetLimits& limits, CookedCpuAssetSet* output, const std::atomic* cancelled) + { + if (!package.IsValid() || !output || output->IsValid() || !limits.MaxObjectBytes || !route.WorkingMemoryBudget()) + { + return VFS::VFSResult::Fail(VFS::VFSError::InvalidPath); + } + Core::Memory::WorkingMemoryCharge fixed = {}; + if (!fixed.TryAcquire(route.WorkingMemoryBudget(), sizeof(BuildContext) + sizeof(CookedObjectSnapshot) + sizeof(fixed))) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + struct CountContext + { + size_t Count = 0; + size_t Maximum = 0; + } count = {0, limits.MaxAssets}; + const auto counted = package.VisitSelectedAssets( + [](void* context, const Cook::CookManifestAsset&) { + auto& state = *static_cast(context); + if (state.Count == state.Maximum) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + ++state.Count; + return VFS::VFSResult::Ok(); + }, + &count, + cancelled); + if (counted.Failed()) + { + return counted; + } + if (count.Count > std::numeric_limits::max() / sizeof(Record)) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + CookedCpuAssetSet candidate = {}; + candidate.m_route = &route; + candidate.m_pool = pool; + candidate.m_capacity = count.Count; + if (count.Count) + { + candidate.m_records = static_cast(route.Allocate(count.Count * sizeof(Record))); + if (!candidate.m_records) + { + return VFS::VFSResult::Fail(VFS::VFSError::OutOfMemory); + } + } + BuildContext context = {&candidate, &package, &limits, cancelled}; + const auto built = package.VisitSelectedAssets(BuildAsset, &context, cancelled); + if (built.Failed()) + { + return built; + } + candidate.m_manifest_digest = package.ManifestDigest(); + candidate.m_scene_uuid = package.Scene()->UUID; + candidate.m_valid = true; + *output = std::move(candidate); + return VFS::VFSResult::Ok(); + } + + const CookedCpuAssetSet::Record* CookedCpuAssetSet::Find(const Cook::CookAssetIdentity& identity) const noexcept + { + if (!m_valid || !m_count) + { + return nullptr; + } + const auto* found = std::lower_bound(m_records, m_records + m_count, identity, [](const Record& record, const Cook::CookAssetIdentity& value) { return record.Version.Identity < value; }); + return found != m_records + m_count && found->Version.Identity == identity ? found : nullptr; + } + + const CookedCpuAssetVersion* CookedCpuAssetSet::VersionAt(size_t index) const noexcept + { + return m_valid && index < m_count ? &m_records[index].Version : nullptr; + } + + const CookedCpuAssetVersion* CookedCpuAssetSet::FindVersion(const Cook::CookAssetIdentity& identity) const noexcept + { + const auto* record = Find(identity); + return record ? &record->Version : nullptr; + } + + const Importers::AssetMesh* CookedCpuAssetSet::FindMesh(const Cook::CookAssetIdentity& identity) const noexcept + { + const auto* record = identity.Kind == Cook::CookAssetKind::Mesh ? Find(identity) : nullptr; + return record ? &static_cast(record->Payload)->Mesh : nullptr; + } + + const Importers::AssetNodeHierarchy* CookedCpuAssetSet::FindHierarchy(const Cook::CookAssetIdentity& identity) const noexcept + { + const auto* record = identity.Kind == Cook::CookAssetKind::Mesh ? Find(identity) : nullptr; + return record ? &static_cast(record->Payload)->Hierarchy : nullptr; + } + + const Codec::MaterialAssetDocument* CookedCpuAssetSet::FindMaterial(const Cook::CookAssetIdentity& identity) const noexcept + { + const auto* record = identity.Kind == Cook::CookAssetKind::Material ? Find(identity) : nullptr; + return record ? static_cast(record->Payload)->Document : nullptr; + } + + const Image::DecodedImage* CookedCpuAssetSet::FindTexture(const Cook::CookAssetIdentity& identity) const noexcept + { + const auto* record = identity.Kind == Cook::CookAssetKind::Texture ? Find(identity) : nullptr; + return record ? &static_cast(record->Payload)->Image : nullptr; + } + + const Codec::EnvironmentMapFileHeader* CookedCpuAssetSet::FindEnvironment(const Cook::CookAssetIdentity& identity) const noexcept + { + const auto* record = identity.Kind == Cook::CookAssetKind::EnvironmentMap ? Find(identity) : nullptr; + return record ? &static_cast(record->Payload)->Header : nullptr; + } + + std::span CookedCpuAssetSet::EnvironmentObject(const Cook::CookAssetIdentity& identity) const noexcept + { + const auto* record = identity.Kind == Cook::CookAssetKind::EnvironmentMap ? Find(identity) : nullptr; + return record ? static_cast(record->Payload)->Object.Bytes() : std::span{}; + } +} // namespace ZEngine::Managers diff --git a/ZEngine/ZEngine/Managers/CookedCpuAssetSet.h b/ZEngine/ZEngine/Managers/CookedCpuAssetSet.h new file mode 100644 index 000000000..5126c626c --- /dev/null +++ b/ZEngine/ZEngine/Managers/CookedCpuAssetSet.h @@ -0,0 +1,93 @@ +#pragma once +#include + +namespace ZEngine::Managers +{ + class CookedCpuClosureCache; + class CookedCpuAssetPool; + /// Independent CPU versions for one selected package closure. These limits + /// are runtime admission policy; they do not inherit the cooker's budget. + struct CookedCpuAssetLimits + { + uint32_t MaxAssets = 0; + size_t MaxObjectBytes = 0; + size_t MaxMeshWorkspaceBytes = 0; + Importers::AssetCodec::MeshAssetReadLimits Mesh = {}; + Importers::ImageCodec::EncodedImageLimits Texture = {}; + Importers::AssetCodec::EnvironmentMapReadLimits Environment = {}; + }; + + struct CookedCpuAssetVersion + { + Cook::CookAssetIdentity Identity = {}; + Cook::Digest256 ObjectDigest = {}; + // Recipe hint; resolve dependency versions within this owner rather + // than treating this unverified key as a content identity. + Cook::Digest256 InputKey = {}; + const Cook::CookAssetFormat* Format = nullptr; + uint64_t ObjectBytes = 0; + }; + + /// Move-only owner of a completely decoded selected closure. The route and + /// its working budget outlive this owner and its borrowers. No registry, + /// renderer, shared cache or GPU state is involved in standalone Capture. + /// A successful capture can outlive its package and root. + class CookedCpuAssetSet final + { + public: + CookedCpuAssetSet() = default; + ~CookedCpuAssetSet(); + CookedCpuAssetSet(const CookedCpuAssetSet&) = delete; + CookedCpuAssetSet& operator=(const CookedCpuAssetSet&) = delete; + CookedCpuAssetSet(CookedCpuAssetSet&&) noexcept; + CookedCpuAssetSet& operator=(CookedCpuAssetSet&&) noexcept; + + /// Worker-only transaction. The route must provide a working budget. + /// Failure releases every candidate allocation and preserves output. + /// Cancellation is observed between assets and around individual reads; + /// synchronous codec calls are allowed to complete before returning. + [[nodiscard]] static Core::VFS::VFSResult Capture(CookedPackageSnapshot&, Core::Memory::IAllocationRoute&, const CookedCpuAssetLimits&, CookedCpuAssetSet*, const std::atomic* cancelled = nullptr); + void Reset() noexcept; + [[nodiscard]] bool IsValid() const noexcept + { + return m_valid; + } + [[nodiscard]] size_t Size() const noexcept + { + return m_count; + } + [[nodiscard]] const Cook::Digest256& ManifestDigest() const noexcept + { + return m_manifest_digest; + } + [[nodiscard]] const uuids::uuid& SceneUUID() const noexcept + { + return m_scene_uuid; + } + [[nodiscard]] const CookedCpuAssetVersion* VersionAt(size_t index) const noexcept; + [[nodiscard]] const CookedCpuAssetVersion* FindVersion(const Cook::CookAssetIdentity&) const noexcept; + [[nodiscard]] const Importers::AssetMesh* FindMesh(const Cook::CookAssetIdentity&) const noexcept; + [[nodiscard]] const Importers::AssetNodeHierarchy* FindHierarchy(const Cook::CookAssetIdentity&) const noexcept; + [[nodiscard]] const Importers::AssetCodec::MaterialAssetDocument* FindMaterial(const Cook::CookAssetIdentity&) const noexcept; + [[nodiscard]] const Importers::ImageCodec::DecodedImage* FindTexture(const Cook::CookAssetIdentity&) const noexcept; + [[nodiscard]] const Importers::AssetCodec::EnvironmentMapFileHeader* FindEnvironment(const Cook::CookAssetIdentity&) const noexcept; + [[nodiscard]] std::span EnvironmentObject(const Cook::CookAssetIdentity&) const noexcept; + + private: + friend class CookedCpuClosureCache; + struct Record; + struct BuildContext; + [[nodiscard]] static Core::VFS::VFSResult CaptureShared(CookedPackageSnapshot&, Core::Memory::IAllocationRoute&, CookedCpuAssetPool&, const CookedCpuAssetLimits&, CookedCpuAssetSet*, const std::atomic*); + [[nodiscard]] static Core::VFS::VFSResult CaptureImpl(CookedPackageSnapshot&, Core::Memory::IAllocationRoute&, CookedCpuAssetPool*, const CookedCpuAssetLimits&, CookedCpuAssetSet*, const std::atomic*); + [[nodiscard]] static Core::VFS::VFSResult BuildAsset(void*, const Cook::CookManifestAsset&); + [[nodiscard]] const Record* Find(const Cook::CookAssetIdentity&) const noexcept; + Core::Memory::IAllocationRoute* m_route = nullptr; + CookedCpuAssetPool* m_pool = nullptr; + Record* m_records = nullptr; + size_t m_count = 0; + size_t m_capacity = 0; + Cook::Digest256 m_manifest_digest = {}; + uuids::uuid m_scene_uuid = {}; + bool m_valid = false; + }; +} // namespace ZEngine::Managers diff --git a/ZEngine/ZEngine/Managers/CookedCpuClosureCache.cpp b/ZEngine/ZEngine/Managers/CookedCpuClosureCache.cpp new file mode 100644 index 000000000..9ae95de28 --- /dev/null +++ b/ZEngine/ZEngine/Managers/CookedCpuClosureCache.cpp @@ -0,0 +1,431 @@ +#include +#include +#include +#include +#include + +namespace ZEngine::Managers +{ + namespace VFS = Core::VFS; + + struct CookedCpuClosureCache::Entry + { + Phase PhaseValue = Phase::Free; + uint64_t Serial = 0; + uint32_t Leases = 0; + Cook::Digest256 Manifest = {}; + uuids::uuid Scene = {}; + std::atomic Cancelled = false; + VFS::VFSError Error = VFS::VFSError::OK; + CookedPackageSnapshot Package = {}; + CookedCpuAssetSet Assets = {}; + }; + + CookedCpuClosureCache::Lease::~Lease() + { + Reset(); + } + + CookedCpuClosureCache::Lease::Lease(Lease&& other) noexcept : m_owner(std::exchange(other.m_owner, nullptr)), m_slot(std::exchange(other.m_slot, 0)), m_serial(std::exchange(other.m_serial, 0)) {} + + CookedCpuClosureCache::Lease& CookedCpuClosureCache::Lease::operator=(Lease&& other) noexcept + { + if (this != &other) + { + Reset(); + m_owner = std::exchange(other.m_owner, nullptr); + m_slot = std::exchange(other.m_slot, 0); + m_serial = std::exchange(other.m_serial, 0); + } + return *this; + } + + bool CookedCpuClosureCache::Lease::Poll(Result* output) const + { + return m_owner && m_owner->Poll(m_slot, m_serial, output); + } + + bool CookedCpuClosureCache::Lease::Retain(Lease* output) const + { + return m_owner && m_owner->Retain(m_slot, m_serial, output); + } + + void CookedCpuClosureCache::Lease::Reset() noexcept + { + if (m_owner) + { + auto* owner = std::exchange(m_owner, nullptr); + owner->Release(m_slot, m_serial); + } + m_slot = 0; + m_serial = 0; + } + + CookedCpuClosureCache::~CookedCpuClosureCache() + { + const bool closed = Shutdown(); + ZENGINE_VALIDATE_ASSERT(closed, "Cooked CPU closure cache destroyed before its leases and worker were drained.") + (void) closed; + } + + bool CookedCpuClosureCache::Initialize(Core::Memory::IAllocationRoute& route, const ECS::SceneComponentSchemaRegistry& schemas, const CookedCpuAssetLimits& limits, uint32_t capacity, uint32_t max_leases_per_closure, uint32_t max_asset_versions) + { + if (!capacity || capacity > MaxClosures || max_asset_versions > MaxAssetVersions || !max_leases_per_closure || !limits.MaxObjectBytes || !route.WorkingMemoryBudget() || !schemas.Frozen() || size_t(capacity) > std::numeric_limits::max() / sizeof(Entry)) + { + return false; + } + std::lock_guard worker_lock(m_worker_mutex); + std::lock_guard lock(m_mutex); + if (m_entries) + { + return false; + } + static_assert(alignof(Entry) <= alignof(std::max_align_t)); + auto* entries = static_cast(route.Allocate(size_t(capacity) * sizeof(Entry))); + if (!entries) + { + return false; + } + for (uint32_t i = 0; i < capacity; ++i) + { + new (entries + i) Entry{}; + } + const auto discard_entries = [&] { + for (uint32_t i = 0; i < capacity; ++i) + { + entries[i].~Entry(); + } + route.Free(entries); + }; + static_assert(alignof(CookedCpuAssetPool) <= alignof(std::max_align_t)); + void* pool_storage = route.Allocate(sizeof(CookedCpuAssetPool)); + if (!pool_storage) + { + discard_entries(); + return false; + } + auto* pool = new (pool_storage) CookedCpuAssetPool{}; + if (!pool->Initialize(route, max_asset_versions)) + { + pool->~CookedCpuAssetPool(); + route.Free(pool_storage); + discard_entries(); + return false; + } + m_route = &route; + m_schemas = &schemas; + m_pool = pool; + m_entries = entries; + m_limits = limits; + m_capacity = capacity; + m_max_leases = max_leases_per_closure; + m_cursor = 0; + m_next_serial = 0; + return true; + } + + bool CookedCpuClosureCache::Request(CookedPackageSnapshot&& package, Lease* output) + { + if (!output || *output || !package.IsValid() || !package.Scene()) + { + return false; + } + std::lock_guard lock(m_mutex); + if (!m_entries || package.Schemas() != m_schemas) + { + return false; + } + const auto digest = package.ManifestDigest(); + const auto scene = package.Scene()->UUID; + uint32_t free_slot = m_capacity; + for (uint32_t i = 0; i < m_capacity; ++i) + { + auto& entry = m_entries[i]; + if (entry.PhaseValue == Phase::Free) + { + if (free_slot == m_capacity) + { + free_slot = i; + } + continue; + } + const bool shareable = entry.PhaseValue == Phase::Queued || entry.PhaseValue == Phase::Building || entry.PhaseValue == Phase::Ready; + if (shareable && !entry.Cancelled.load(std::memory_order_acquire) && entry.Manifest == digest && entry.Scene == scene) + { + if (entry.Leases == m_max_leases) + { + return false; + } + ++entry.Leases; + output->m_owner = this; + output->m_slot = i; + output->m_serial = entry.Serial; + return true; + } + } + if (free_slot == m_capacity || m_next_serial == std::numeric_limits::max()) + { + return false; + } + auto& entry = m_entries[free_slot]; + entry.Serial = ++m_next_serial; + entry.Leases = 1; + entry.Manifest = digest; + entry.Scene = scene; + entry.Cancelled.store(false, std::memory_order_relaxed); + entry.Error = VFS::VFSError::OK; + entry.Package = std::move(package); + entry.PhaseValue = Phase::Queued; + output->m_owner = this; + output->m_slot = free_slot; + output->m_serial = entry.Serial; + return true; + } + + bool CookedCpuClosureCache::Retain(uint32_t slot, uint64_t serial, Lease* output) + { + if (!output || *output) + { + return false; + } + std::lock_guard lock(m_mutex); + if (!m_entries || slot >= m_capacity) + { + return false; + } + auto& entry = m_entries[slot]; + const bool shareable = entry.PhaseValue == Phase::Queued || entry.PhaseValue == Phase::Building || entry.PhaseValue == Phase::Ready || entry.PhaseValue == Phase::Failed; + if (entry.Serial != serial || !entry.Leases || entry.Leases >= m_max_leases || !shareable || entry.Cancelled.load(std::memory_order_acquire)) + { + return false; + } + ++entry.Leases; + output->m_owner = this; + output->m_slot = slot; + output->m_serial = serial; + return true; + } + + bool CookedCpuClosureCache::Poll(uint32_t slot, uint64_t serial, Result* output) const + { + if (!output) + { + return false; + } + std::lock_guard lock(m_mutex); + if (!m_entries || slot >= m_capacity) + { + return false; + } + const auto& entry = m_entries[slot]; + if (entry.Serial != serial || !entry.Leases) + { + return false; + } + Result result = {}; + switch (entry.PhaseValue) + { + case Phase::Queued: + case Phase::Building: + break; + case Phase::Ready: + result.StateValue = State::Ready; + result.Assets = &entry.Assets; + break; + case Phase::Failed: + result.StateValue = State::Failed; + result.Error = entry.Error; + break; + default: + return false; + } + *output = result; + return true; + } + + void CookedCpuClosureCache::Release(uint32_t slot, uint64_t serial) noexcept + { + std::lock_guard lock(m_mutex); + if (!m_entries || slot >= m_capacity) + { + return; + } + auto& entry = m_entries[slot]; + if (entry.Serial != serial || !entry.Leases) + { + return; + } + if (--entry.Leases) + { + return; + } + if (entry.PhaseValue == Phase::Queued || entry.PhaseValue == Phase::Building) + { + entry.Cancelled.store(true, std::memory_order_release); + } + else if (entry.PhaseValue == Phase::Ready || entry.PhaseValue == Phase::Failed) + { + entry.PhaseValue = Phase::Retiring; + } + } + + bool CookedCpuClosureCache::CancelUnsharedBuild(CancellationToken token) noexcept + { + std::lock_guard lock(m_mutex); + if (!m_entries || token.Owner != this || !token.Serial || token.Slot >= m_capacity) + return false; + auto& entry = m_entries[token.Slot]; + if (entry.Serial != token.Serial || entry.Leases != 1 || entry.PhaseValue != Phase::Building) + return false; + entry.Cancelled.store(true, std::memory_order_release); + return true; + } + + bool CookedCpuClosureCache::TryGetStatistics(Statistics* output) + { + if (!output) + return false; + std::unique_lock worker_lock(m_worker_mutex, std::try_to_lock); + if (!worker_lock.owns_lock()) + return false; + std::lock_guard lock(m_mutex); + if (!m_entries || !m_pool) + return false; + Statistics result = {}; + result.AssetVersions = m_pool->m_live; + for (uint32_t i = 0; i < m_capacity; ++i) + { + const auto& entry = m_entries[i]; + if (entry.PhaseValue != Phase::Free) + ++result.Closures; + result.Leases += entry.Leases; + if (entry.PhaseValue == Phase::Retiring || entry.PhaseValue == Phase::Cleaning) + ++result.Retiring; + } + *output = result; + return true; + } + + bool CookedCpuClosureCache::PumpOne(const std::atomic* owner_cancelled) + { + std::unique_lock worker_lock(m_worker_mutex, std::try_to_lock); + if (!worker_lock.owns_lock()) + { + return false; + } + uint32_t selected = 0; + bool build = false; + { + std::lock_guard lock(m_mutex); + if (!m_entries) + { + return false; + } + selected = m_capacity; + for (uint32_t offset = 0; offset < m_capacity; ++offset) + { + const uint32_t index = static_cast((uint64_t(m_cursor) + offset) % m_capacity); + auto& entry = m_entries[index]; + if (entry.PhaseValue == Phase::Retiring || entry.PhaseValue == Phase::Queued) + { + selected = index; + m_cursor = (index + 1) % m_capacity; + build = entry.PhaseValue == Phase::Queued && entry.Leases && !entry.Cancelled.load(std::memory_order_acquire); + entry.PhaseValue = build ? Phase::Building : Phase::Cleaning; + if (build && entry.Leases == 1 && owner_cancelled && owner_cancelled->load(std::memory_order_acquire)) + entry.Cancelled.store(true, std::memory_order_release); + break; + } + } + if (selected == m_capacity) + { + return false; + } + } + auto& entry = m_entries[selected]; + if (build) + { + const auto result = CookedCpuAssetSet::CaptureShared(entry.Package, *m_route, *m_pool, m_limits, &entry.Assets, &entry.Cancelled); + { + std::lock_guard lock(m_mutex); + if (!entry.Leases) + { + entry.PhaseValue = Phase::Cleaning; + } + else if (entry.Cancelled.load(std::memory_order_acquire)) + { + entry.Error = VFS::VFSError::Cancelled; + entry.PhaseValue = Phase::Failed; + } + else if (result.Failed()) + { + entry.Error = result.Error(); + entry.PhaseValue = Phase::Failed; + } + else + { + entry.PhaseValue = Phase::Ready; + } + } + } + // These objects are touched only by this worker. Never free a closure or + // its rooted package while holding the producer-facing state lock. + entry.Package.Reset(); + bool cleaning = false; + { + std::lock_guard lock(m_mutex); + cleaning = entry.PhaseValue == Phase::Cleaning; + } + if (cleaning) + { + entry.Assets.Reset(); + std::lock_guard lock(m_mutex); + entry.Manifest = {}; + entry.Scene = {}; + entry.Error = VFS::VFSError::OK; + entry.PhaseValue = Phase::Free; + } + return true; + } + + bool CookedCpuClosureCache::Shutdown() + { + std::unique_lock worker_lock(m_worker_mutex, std::try_to_lock); + if (!worker_lock.owns_lock()) + { + return false; + } + std::lock_guard lock(m_mutex); + if (!m_entries) + { + return true; + } + for (uint32_t i = 0; i < m_capacity; ++i) + { + if (m_entries[i].PhaseValue != Phase::Free || m_entries[i].Leases) + { + return false; + } + } + if (!m_pool || m_pool->m_live) + { + return false; + } + for (uint32_t i = 0; i < m_capacity; ++i) + { + m_entries[i].~Entry(); + } + (void) m_pool->Shutdown(); + m_pool->~CookedCpuAssetPool(); + m_route->Free(m_pool); + m_route->Free(m_entries); + m_entries = nullptr; + m_route = nullptr; + m_schemas = nullptr; + m_pool = nullptr; + m_capacity = 0; + m_max_leases = 0; + m_cursor = 0; + return true; + } +} // namespace ZEngine::Managers diff --git a/ZEngine/ZEngine/Managers/CookedCpuClosureCache.h b/ZEngine/ZEngine/Managers/CookedCpuClosureCache.h new file mode 100644 index 000000000..76a727a32 --- /dev/null +++ b/ZEngine/ZEngine/Managers/CookedCpuClosureCache.h @@ -0,0 +1,139 @@ +#pragma once +#include +#include +#include + +namespace ZEngine::Managers +{ + /// Bounded asynchronous sharing of identical verified scene closures and + /// immutable asset versions across different closures. One worker builds + /// closures sequentially; only completed versions are shared across them. + /// The supplied allocation route is used only by initialization, one worker + /// at a time and quiescent shutdown. It and every package's allocator/schema + /// registry outlive the cache, its workers, and every lease. Package root + /// providers and allocators shared by concurrent callers must support that + /// concurrency or be synchronized by their owner. + class CookedCpuClosureCache final + { + public: + static constexpr uint32_t MaxClosures = 16; + static constexpr uint32_t MaxAssetVersions = 65536; + enum class State : uint8_t + { + Pending, + Ready, + Failed, + }; + + struct Result + { + State StateValue = State::Pending; + Core::VFS::VFSError Error = Core::VFS::VFSError::OK; + const CookedCpuAssetSet* Assets = nullptr; + }; + + struct CancellationToken + { + const CookedCpuClosureCache* Owner = nullptr; + uint32_t Slot = 0; + uint64_t Serial = 0; + }; + + /// A lease is exclusive to its owner, but different leases may be + /// polled/released on different threads. An Assets borrow lasts only + /// while this lease remains live; never reset it concurrently with use. + class Lease final + { + public: + Lease() = default; + ~Lease(); + Lease(const Lease&) = delete; + Lease& operator=(const Lease&) = delete; + Lease(Lease&&) noexcept; + Lease& operator=(Lease&&) noexcept; + [[nodiscard]] bool Poll(Result*) const; + /// Retain the same closure without reopening its package. The + /// output must be empty; bounded admission can reject the retain. + /// As with Poll, synchronize against resetting this lease. + [[nodiscard]] bool Retain(Lease* output) const; + /// Copy only while this lease is exclusively owned; the token can + /// be published to another thread and does not retain the lease or + /// the cache; both must still outlive token use. + [[nodiscard]] CancellationToken CancellationIdentity() const noexcept + { + return {m_owner, m_slot, m_serial}; + } + explicit operator bool() const noexcept + { + return m_owner != nullptr; + } + void Reset() noexcept; + + private: + friend class CookedCpuClosureCache; + CookedCpuClosureCache* m_owner = nullptr; + uint32_t m_slot = 0; + uint64_t m_serial = 0; + }; + + CookedCpuClosureCache() = default; + ~CookedCpuClosureCache(); + CookedCpuClosureCache(const CookedCpuClosureCache&) = delete; + CookedCpuClosureCache& operator=(const CookedCpuClosureCache&) = delete; + + /// The route must provide a working-memory budget. Capacities and asset + /// limits remain fixed until shutdown, so shared requests cannot weaken + /// one another's admission policy. + [[nodiscard]] bool Initialize(Core::Memory::IAllocationRoute&, const ECS::SceneComponentSchemaRegistry&, const CookedCpuAssetLimits&, uint32_t capacity, uint32_t max_leases_per_closure, uint32_t max_asset_versions); + /// Coalesces by captured manifest digest and selected scene UUID. On a + /// new request the package is consumed; on a hit it remains with the + /// caller. False means invalid input or bounded admission backpressure. + [[nodiscard]] bool Request(CookedPackageSnapshot&& package, Lease* output); + /// Worker-only, nonblocking admission to the single cache worker. Runs + /// one queued build or retirement; false means no work or another worker + /// is active. Call again until idle, including during shutdown. + [[nodiscard]] bool PumpOne(const std::atomic* owner_cancelled = nullptr); + /// Signal a building closure only if this exact lease is its sole + /// owner. Shared consumers must not be cancelled on another's behalf. + [[nodiscard]] bool CancelUnsharedBuild(CancellationToken token) noexcept; + struct Statistics + { + uint32_t Closures = 0; + uint32_t Leases = 0; + uint32_t Retiring = 0; + uint32_t AssetVersions = 0; + }; + /// Nonblocking snapshot. False leaves output unchanged if the worker is busy. + [[nodiscard]] bool TryGetStatistics(Statistics* output); + /// Quiescent operation. Fails without modifying the cache while leases, + /// queued work, retirement or a worker remain. Drain using PumpOne first. + [[nodiscard]] bool Shutdown(); + + private: + enum class Phase : uint8_t + { + Free, + Queued, + Building, + Ready, + Failed, + Retiring, + Cleaning + }; + struct Entry; + [[nodiscard]] bool Poll(uint32_t slot, uint64_t serial, Result*) const; + [[nodiscard]] bool Retain(uint32_t slot, uint64_t serial, Lease* output); + void Release(uint32_t slot, uint64_t serial) noexcept; + mutable std::mutex m_mutex = {}; + std::mutex m_worker_mutex = {}; + Core::Memory::IAllocationRoute* m_route = nullptr; + const ECS::SceneComponentSchemaRegistry* m_schemas = nullptr; + CookedCpuAssetPool* m_pool = nullptr; + Entry* m_entries = nullptr; + CookedCpuAssetLimits m_limits = {}; + uint64_t m_next_serial = 0; + uint32_t m_capacity = 0; + uint32_t m_max_leases = 0; + uint32_t m_cursor = 0; + }; +} // namespace ZEngine::Managers diff --git a/ZEngine/ZEngine/Managers/CookedObjectSnapshot.cpp b/ZEngine/ZEngine/Managers/CookedObjectSnapshot.cpp new file mode 100644 index 000000000..92a9f67c9 --- /dev/null +++ b/ZEngine/ZEngine/Managers/CookedObjectSnapshot.cpp @@ -0,0 +1,177 @@ +#include +#include +#include +#include +#include + +namespace ZEngine::Managers +{ + namespace VFS = Core::VFS; + + namespace + { + bool IsCancelled(const std::atomic* cancelled) + { + return cancelled && cancelled->load(std::memory_order_acquire); + } + + VFS::VFSResult CaptureBytes(VFS::IVFSFile& file, const Cook::CookObjectDescriptor& object, uint8_t* bytes, const std::atomic* cancelled) + { + const auto before = file.Stat(); + if (before.Failed()) + { + return VFS::VFSResult::Fail(before.Error()); + } + if (before.Value().IsDirectory || before.Value().SizeBytes != object.Size) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + Core::Hash::SHA256 hash = {}; + constexpr size_t read_slice = ZKilo(64); + for (uint64_t offset = 0; offset < object.Size;) + { + if (IsCancelled(cancelled)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Cancelled); + } + const auto count = static_cast(std::min(read_slice, object.Size - offset)); + const auto read = file.Read({bytes + static_cast(offset), count}, offset); + if (read.Failed() || !read.Value() || read.Value() > count) + { + return VFS::VFSResult::Fail(read.Failed() ? read.Error() : VFS::VFSError::Corrupted); + } + const auto hashed = hash.Update({bytes + static_cast(offset), read.Value()}); + if (hashed.Failed()) + { + return hashed; + } + offset += read.Value(); + } + if (IsCancelled(cancelled)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Cancelled); + } + uint8_t trailing = 0; + const auto eof = file.Read({&trailing, 1}, object.Size); + if (eof.Failed() || eof.Value()) + { + return VFS::VFSResult::Fail(eof.Failed() ? eof.Error() : VFS::VFSError::Corrupted); + } + const auto after = file.Stat(); + if (after.Failed()) + { + return VFS::VFSResult::Fail(after.Error()); + } + if (after.Value().IsDirectory || after.Value().SizeBytes != object.Size) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + Cook::Digest256 digest = {}; + const auto finalized = hash.Finalize(&digest); + if (finalized.Failed()) + { + return finalized; + } + return digest == object.Digest ? VFS::VFSResult::Ok() : VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + } // namespace + + CookedObjectSnapshot::~CookedObjectSnapshot() + { + Reset(); + } + + CookedObjectSnapshot::CookedObjectSnapshot(CookedObjectSnapshot&& other) noexcept : m_route(std::exchange(other.m_route, nullptr)), m_bytes(std::exchange(other.m_bytes, nullptr)), m_size(std::exchange(other.m_size, 0)), m_descriptor(std::exchange(other.m_descriptor, {})) {} + + CookedObjectSnapshot& CookedObjectSnapshot::operator=(CookedObjectSnapshot&& other) noexcept + { + if (this != &other) + { + Reset(); + m_route = std::exchange(other.m_route, nullptr); + m_bytes = std::exchange(other.m_bytes, nullptr); + m_size = std::exchange(other.m_size, 0); + m_descriptor = std::exchange(other.m_descriptor, {}); + } + return *this; + } + + void CookedObjectSnapshot::Reset() noexcept + { + if (m_route) + { + m_route->Free(m_bytes); + } + m_route = nullptr; + m_bytes = nullptr; + m_size = 0; + m_descriptor = {}; + } + + Importers::AssetCodec::AssetByteSource CookedObjectSnapshot::Source() const noexcept + { + return { + const_cast(this), + [](void* context, Core::Containers::ArrayView destination, uint64_t offset) { + const auto& owner = *static_cast(context); + if ((destination.size() && !destination.data()) || offset > owner.m_size) + { + return VFS::VFSResult::Fail(VFS::VFSError::InvalidPath); + } + const size_t count = std::min(destination.size(), owner.m_size - static_cast(offset)); + if (count && Helpers::secure_memcpy(destination.data(), destination.size(), owner.m_bytes + static_cast(offset), count) != Helpers::MEMORY_OP_SUCCESS) + { + return VFS::VFSResult::Fail(VFS::VFSError::IOError); + } + return VFS::VFSResult::Ok(count); + }, + m_size}; + } + + VFS::VFSResult CookedObjectSnapshot::Capture(const Cook::CookObjectReadFns& reader, const Cook::CookObjectDescriptor& object, Core::Containers::StringView extension, Core::Memory::IAllocationRoute& route, size_t max_bytes, CookedObjectSnapshot* output, const std::atomic* cancelled) + { + if (!output || output->m_bytes || !reader.Context || !reader.Open || !reader.CloseChecked || !max_bytes) + { + return VFS::VFSResult::Fail(VFS::VFSError::InvalidPath); + } + if (!Cook::IsCookObjectDescriptorValid(object, extension)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + if (object.Size > max_bytes || object.Size > std::numeric_limits::max()) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + if (IsCancelled(cancelled)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Cancelled); + } + Core::Memory::WorkingMemoryCharge fixed = {}; + if (!fixed.TryAcquire(route.WorkingMemoryBudget(), Core::Hash::SHA256::FixedWorkingBytes() + sizeof(fixed))) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + const auto opened = reader.Open(reader.Context, object.Path); + if (opened.Failed()) + { + return VFS::VFSResult::Fail(opened.Error()); + } + if (!opened.Value()) + { + return VFS::VFSResult::Fail(VFS::VFSError::IOError); + } + CookedObjectSnapshot candidate = {}; + candidate.m_route = &route; + candidate.m_size = static_cast(object.Size); + candidate.m_descriptor = object; + candidate.m_bytes = static_cast(route.Allocate(candidate.m_size)); + const auto captured = candidate.m_bytes ? CaptureBytes(*opened.Value(), object, candidate.m_bytes, cancelled) : VFS::VFSResult::Fail(VFS::VFSError::OutOfMemory); + const auto closed = reader.CloseChecked(reader.Context, opened.Value()); + if (captured.Failed() || closed.Failed() || IsCancelled(cancelled)) + { + return VFS::VFSResult::Fail(captured.Failed() ? captured.Error() : closed.Failed() ? closed.Error() : VFS::VFSError::Cancelled); + } + *output = std::move(candidate); + return VFS::VFSResult::Ok(); + } +} // namespace ZEngine::Managers diff --git a/ZEngine/ZEngine/Managers/CookedObjectSnapshot.h b/ZEngine/ZEngine/Managers/CookedObjectSnapshot.h new file mode 100644 index 000000000..f8d04dd6a --- /dev/null +++ b/ZEngine/ZEngine/Managers/CookedObjectSnapshot.h @@ -0,0 +1,48 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace ZEngine::Managers +{ + /// Private verified bytes for one cooked object. The allocation route must + /// outlive this owner and every borrower of Bytes(). This synchronous helper + /// belongs on a load worker; it does not open source assets or publish a scene. + class CookedObjectSnapshot final + { + public: + CookedObjectSnapshot() = default; + ~CookedObjectSnapshot(); + CookedObjectSnapshot(const CookedObjectSnapshot&) = delete; + CookedObjectSnapshot& operator=(const CookedObjectSnapshot&) = delete; + CookedObjectSnapshot(CookedObjectSnapshot&& other) noexcept; + CookedObjectSnapshot& operator=(CookedObjectSnapshot&& other) noexcept; + + /// Copies and hashes the exact bytes read from one open handle. Path, + /// size, EOF, digest and checked-close must all succeed before output + /// changes. max_bytes is an admitted per-object bound; the route owns + /// allocation accounting. An already populated output is refused so + /// existing borrowed views cannot be invalidated implicitly. + [[nodiscard]] static Core::VFS::VFSResult Capture(const Cook::CookObjectReadFns& reader, const Cook::CookObjectDescriptor& object, Core::Containers::StringView extension, Core::Memory::IAllocationRoute& route, size_t max_bytes, CookedObjectSnapshot* output, const std::atomic* cancelled = nullptr); + + void Reset() noexcept; + [[nodiscard]] std::span Bytes() const noexcept + { + return {m_bytes, m_size}; + } + [[nodiscard]] const Cook::CookObjectDescriptor& Descriptor() const noexcept + { + return m_descriptor; + } + /// Borrowed reader; moving/resetting this owner invalidates its context. + [[nodiscard]] Importers::AssetCodec::AssetByteSource Source() const noexcept; + + private: + Core::Memory::IAllocationRoute* m_route = nullptr; + uint8_t* m_bytes = nullptr; + size_t m_size = 0; + Cook::CookObjectDescriptor m_descriptor = {}; + }; +} // namespace ZEngine::Managers diff --git a/ZEngine/ZEngine/Managers/CookedPackageSnapshot.cpp b/ZEngine/ZEngine/Managers/CookedPackageSnapshot.cpp new file mode 100644 index 000000000..aff3f3090 --- /dev/null +++ b/ZEngine/ZEngine/Managers/CookedPackageSnapshot.cpp @@ -0,0 +1,485 @@ +#include +#include +#include +#include +#include +#include + +namespace ZEngine::Managers +{ + namespace VFS = Core::VFS; + + namespace + { + class TemporaryBlock + { + public: + ~TemporaryBlock() + { + if (m_data) + { + m_allocator->Free(m_data); + } + } + VFS::VFSResult Allocate(Cook::ICookAllocator& allocator, size_t bytes, size_t alignment) + { + if (!bytes || m_data) + { + return VFS::VFSResult::Fail(VFS::VFSError::InvalidPath); + } + const auto allocation = allocator.Allocate(bytes, alignment, Cook::CookMemoryCategory::StagingManifest); + if (allocation.Failed()) + { + return VFS::VFSResult::Fail(allocation.Error()); + } + if (!allocation.Value()) + { + return VFS::VFSResult::Fail(VFS::VFSError::OutOfMemory); + } + m_allocator = &allocator; + m_data = allocation.Value(); + return VFS::VFSResult::Ok(); + } + void* Data() const noexcept + { + return m_data; + } + void* Release() noexcept + { + return std::exchange(m_data, nullptr); + } + + private: + Cook::ICookAllocator* m_allocator = nullptr; + void* m_data = nullptr; + }; + + bool IsCancelled(const std::atomic* cancelled) + { + return cancelled && cancelled->load(std::memory_order_acquire); + } + + VFS::VFSResult ReadManifest(VFS::IVFSFile& file, const Cook::CookLimits& limits, Cook::ICookAllocator& allocator, TemporaryBlock& bytes, size_t* size, const std::atomic* cancelled) + { + const auto before = file.Stat(); + if (before.Failed()) + { + return VFS::VFSResult::Fail(before.Error()); + } + if (before.Value().IsDirectory || !before.Value().SizeBytes) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + if (before.Value().SizeBytes > limits.MaxManifestBytes || before.Value().SizeBytes > std::numeric_limits::max()) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + const size_t length = static_cast(before.Value().SizeBytes); + const auto allocated = bytes.Allocate(allocator, length, alignof(uint8_t)); + if (allocated.Failed()) + { + return allocated; + } + auto* buffer = static_cast(bytes.Data()); + constexpr size_t read_slice = ZKilo(64); + for (size_t offset = 0; offset < length;) + { + if (IsCancelled(cancelled)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Cancelled); + } + const size_t count = std::min(read_slice, length - offset); + const auto read = file.Read({buffer + offset, count}, offset); + if (read.Failed() || !read.Value() || read.Value() > count) + { + return VFS::VFSResult::Fail(read.Failed() ? read.Error() : VFS::VFSError::Corrupted); + } + offset += read.Value(); + } + uint8_t trailing = 0; + const auto eof = file.Read({&trailing, 1}, length); + if (eof.Failed() || eof.Value()) + { + return VFS::VFSResult::Fail(eof.Failed() ? eof.Error() : VFS::VFSError::Corrupted); + } + const auto after = file.Stat(); + if (after.Failed()) + { + return VFS::VFSResult::Fail(after.Error()); + } + if (after.Value().IsDirectory || after.Value().SizeBytes != length) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + *size = length; + return VFS::VFSResult::Ok(); + } + + VFS::VFSResult ComputeSchemaKey(const ECS::SceneComponentSchemaRegistry& schemas, Cook::ICookAllocator& allocator, Cook::Digest256* result) + { + const auto words = Cook::CookSchemaKeyWorkspaceWords(schemas); + if (words.Failed()) + { + return VFS::VFSResult::Fail(words.Error()); + } + if (words.Value() > std::numeric_limits::max() / sizeof(uint32_t)) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + TemporaryBlock workspace = {}; + if (words.Value()) + { + const auto allocated = workspace.Allocate(allocator, words.Value() * sizeof(uint32_t), alignof(uint32_t)); + if (allocated.Failed()) + { + return allocated; + } + } + return Cook::ComputeCookSchemaKey(allocator, schemas, {static_cast(workspace.Data()), words.Value()}, result); + } + + VFS::VFSResult BuildSelection(const Cook::CookManifest& manifest, const uuids::uuid& scene, Cook::ICookAllocator& allocator, TemporaryBlock& selected, const std::atomic* cancelled) + { + const auto* scene_record = manifest.FindScene(scene); + if (!scene_record) + { + return VFS::VFSResult::Fail(VFS::VFSError::NotFound); + } + const auto assets = manifest.Assets(); + if (assets.empty()) + { + return VFS::VFSResult::Ok(); + } + if (assets.size() > std::numeric_limits::max() / sizeof(uint32_t)) + { + return VFS::VFSResult::Fail(VFS::VFSError::SizeLimitExceeded); + } + auto status = selected.Allocate(allocator, assets.size(), alignof(uint8_t)); + if (status.Failed()) + { + return status; + } + TemporaryBlock queue = {}; + status = queue.Allocate(allocator, assets.size() * sizeof(uint32_t), alignof(uint32_t)); + if (status.Failed()) + { + return status; + } + auto* flags = static_cast(selected.Data()); + auto* pending = static_cast(queue.Data()); + (void) Helpers::secure_memset(flags, 0, assets.size(), assets.size()); + size_t head = 0; + size_t tail = 0; + const auto add = [&](const Cook::CookAssetIdentity& identity) { + const auto* asset = manifest.FindAsset(identity); + if (!asset) + { + return false; + } + const auto index = static_cast(asset - assets.data()); + if (!flags[index]) + { + flags[index] = 1; + pending[tail++] = static_cast(index); + } + return true; + }; + const auto scene_index = static_cast(scene_record - manifest.Scenes().data()); + for (const auto& root : manifest.Roots(scene_index)) + { + if (!add(root)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + } + while (head < tail) + { + if (IsCancelled(cancelled)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Cancelled); + } + for (const auto& dependency : manifest.Dependencies(pending[head++])) + { + if (!add(dependency.Identity)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + } + } + return VFS::VFSResult::Ok(); + } + } // namespace + + CookedPackageSnapshot::~CookedPackageSnapshot() + { + Reset(); + } + + CookedPackageSnapshot::CookedPackageSnapshot(CookedPackageSnapshot&& other) noexcept : m_root(std::move(other.m_root)), m_manifest(std::move(other.m_manifest)), m_allocator(std::exchange(other.m_allocator, nullptr)), m_schemas(std::exchange(other.m_schemas, nullptr)), m_selected(std::exchange(other.m_selected, nullptr)), m_scene(std::exchange(other.m_scene, {})), m_digest(std::exchange(other.m_digest, {})) {} + + CookedPackageSnapshot& CookedPackageSnapshot::operator=(CookedPackageSnapshot&& other) noexcept + { + if (this != &other) + { + Reset(); + m_root = std::move(other.m_root); + m_manifest = std::move(other.m_manifest); + m_allocator = std::exchange(other.m_allocator, nullptr); + m_schemas = std::exchange(other.m_schemas, nullptr); + m_selected = std::exchange(other.m_selected, nullptr); + m_scene = std::exchange(other.m_scene, {}); + m_digest = std::exchange(other.m_digest, {}); + } + return *this; + } + + void CookedPackageSnapshot::Reset() noexcept + { + if (m_selected) + { + m_allocator->Free(m_selected); + } + m_selected = nullptr; + m_manifest.Reset(); + m_root.Reset(); + m_allocator = nullptr; + m_schemas = nullptr; + m_scene = {}; + m_digest = {}; + } + + VFS::VFSResult CookedPackageSnapshot::Capture(VFS::VFSReadRootLease&& root, const uuids::uuid& scene, const ECS::SceneComponentSchemaRegistry& schemas, const Cook::CookLimits& limits, Cook::ICookAllocator& allocator, Cook::CookDiagnostics* diagnostics, CookedPackageSnapshot* output, const std::atomic* cancelled) + { + if (!output || output->IsValid() || !root.IsValid() || scene.is_nil() || !schemas.Frozen()) + { + return VFS::VFSResult::Fail(VFS::VFSError::InvalidPath); + } + const auto validated = Cook::ValidateCookLimits(limits); + if (validated.Failed()) + { + return validated; + } + if (IsCancelled(cancelled)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Cancelled); + } + Cook::Digest256 schema_key = {}; + const auto schema = ComputeSchemaKey(schemas, allocator, &schema_key); + if (schema.Failed()) + { + return schema; + } + const auto path = VFS::VFSRelativePath::Parse(Cook::CookManifestName); + if (path.Failed()) + { + return VFS::VFSResult::Fail(path.Error()); + } + const auto opened = root.Open(path.Value()); + if (opened.Failed() || !opened.Value()) + { + return VFS::VFSResult::Fail(opened.Failed() ? opened.Error() : VFS::VFSError::IOError); + } + TemporaryBlock bytes = {}; + size_t size = 0; + const auto read = ReadManifest(*opened.Value(), limits, allocator, bytes, &size, cancelled); + const auto closed = root.CloseChecked(opened.Value()); + if (read.Failed() || closed.Failed()) + { + return VFS::VFSResult::Fail(read.Failed() ? read.Error() : closed.Error()); + } + if (IsCancelled(cancelled)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Cancelled); + } + Cook::CookMemoryCharge fixed = {}; + const auto charged = allocator.TryCharge(Core::Hash::SHA256::FixedWorkingBytes() + sizeof(fixed), Cook::CookMemoryCategory::StagingManifest, &fixed); + if (charged.Failed()) + { + return charged; + } + Core::Hash::SHA256 hash = {}; + Cook::Digest256 digest = {}; + const auto hashed = hash.Update({static_cast(bytes.Data()), size}); + if (hashed.Failed()) + { + return hashed; + } + const auto finalized = hash.Finalize(&digest); + if (finalized.Failed()) + { + return finalized; + } + CookedPackageSnapshot candidate = {}; + const auto parsed = candidate.m_manifest.Read({static_cast(bytes.Data()), size}, Cook::CookTarget::DesktopVulkanLE64, limits, allocator, diagnostics); + if (parsed.Failed() || parsed.Value() != Cook::CookManifestReadStatus::Valid) + { + return VFS::VFSResult::Fail(parsed.Failed() ? parsed.Error() : VFS::VFSError::Corrupted); + } + if (candidate.m_manifest.Recipe().SchemaKey != schema_key) + { + return VFS::VFSResult::Fail(VFS::VFSError::Unsupported); + } + TemporaryBlock selected = {}; + const auto selection = BuildSelection(candidate.m_manifest, scene, allocator, selected, cancelled); + if (selection.Failed()) + { + return selection; + } + if (IsCancelled(cancelled)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Cancelled); + } + candidate.m_allocator = &allocator; + candidate.m_schemas = &schemas; + candidate.m_selected = static_cast(selected.Release()); + candidate.m_scene = scene; + candidate.m_digest = digest; + candidate.m_root = std::move(root); + *output = std::move(candidate); + return VFS::VFSResult::Ok(); + } + + const Cook::CookManifestScene* CookedPackageSnapshot::Scene() const noexcept + { + return IsValid() ? m_manifest.FindScene(m_scene) : nullptr; + } + + const Cook::CookManifestAsset* CookedPackageSnapshot::FindSelectedAsset(const Cook::CookAssetIdentity& identity) const noexcept + { + const auto* asset = IsValid() ? m_manifest.FindAsset(identity) : nullptr; + if (!asset) + { + return nullptr; + } + const auto index = static_cast(asset - m_manifest.Assets().data()); + return m_selected && m_selected[index] ? asset : nullptr; + } + + VFS::VFSResult CookedPackageSnapshot::VisitSelectedAssets(AssetVisitor visitor, void* context, const std::atomic* cancelled) const + { + if (!IsValid() || !visitor) + { + return VFS::VFSResult::Fail(VFS::VFSError::InvalidPath); + } + const auto assets = m_manifest.Assets(); + for (size_t i = 0; i < assets.size(); ++i) + { + if (IsCancelled(cancelled)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Cancelled); + } + if (m_selected && m_selected[i]) + { + const auto result = visitor(context, assets[i]); + if (result.Failed()) + { + return result; + } + } + } + return IsCancelled(cancelled) ? VFS::VFSResult::Fail(VFS::VFSError::Cancelled) : VFS::VFSResult::Ok(); + } + + Core::Containers::ArrayView CookedPackageSnapshot::Dependencies(const Cook::CookAssetIdentity& identity) const noexcept + { + const auto* asset = FindSelectedAsset(identity); + return asset ? m_manifest.Dependencies(static_cast(asset - m_manifest.Assets().data())) : Core::Containers::ArrayView{}; + } + + VFS::VFSResult CookedPackageSnapshot::ValidateScene(const ECS::SceneDocument& document, const ECS::SceneComponentSchemaRegistry& schemas, ECS::SceneDiagnostics* diagnostics, const std::atomic* cancelled) const + { + const auto* scene = Scene(); + if (!scene || !document.Settings() || document.Settings()->UUID != m_scene || &schemas != m_schemas || !schemas.Frozen()) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + if (IsCancelled(cancelled)) + { + return VFS::VFSResult::Fail(VFS::VFSError::Cancelled); + } + const auto roots = m_manifest.Roots(static_cast(scene - m_manifest.Scenes().data())); + TemporaryBlock seen = {}; + if (!roots.empty()) + { + const auto allocated = seen.Allocate(*m_allocator, roots.size(), alignof(uint8_t)); + if (allocated.Failed()) + { + return allocated; + } + (void) Helpers::secure_memset(seen.Data(), 0, roots.size(), roots.size()); + } + struct VisitorState + { + Core::Containers::ArrayView Roots = {}; + uint8_t* Seen = nullptr; + const std::atomic* Cancelled = nullptr; + VFS::VFSError Error = VFS::VFSError::OK; + } state = {roots, static_cast(seen.Data()), cancelled}; + const auto visit = [](void* context, const ECS::SceneReferenceOccurrence& occurrence) { + auto& state = *static_cast(context); + if (IsCancelled(state.Cancelled)) + { + state.Error = VFS::VFSError::Cancelled; + return false; + } + const auto& reference = occurrence.Reference; + if (reference.Kind == ECS::SceneReferenceKind::Entity) + { + return true; + } + if (reference.Kind != ECS::SceneReferenceKind::Asset || !reference.AssetKindKey) + { + state.Error = VFS::VFSError::Corrupted; + return false; + } + const auto kind = Cook::ParseCookAssetKind(reference.AssetKindKey); + if (kind == Cook::CookAssetKind::Unknown) + { + state.Error = VFS::VFSError::Corrupted; + return false; + } + if (reference.UUID.is_nil()) + { + return true; + } + if (state.Roots.empty()) + { + state.Error = VFS::VFSError::Corrupted; + return false; + } + const Cook::CookAssetIdentity identity = {kind, reference.UUID}; + const auto root = std::lower_bound(state.Roots.begin(), state.Roots.end(), identity); + if (root == state.Roots.end() || *root != identity) + { + state.Error = VFS::VFSError::Corrupted; + return false; + } + state.Seen[static_cast(root - state.Roots.begin())] = 1; + return true; + }; + if (!document.EnumerateReferences(schemas, visit, &state, diagnostics)) + { + return VFS::VFSResult::Fail(state.Error == VFS::VFSError::OK ? VFS::VFSError::Corrupted : state.Error); + } + for (size_t i = 0; i < roots.size(); ++i) + { + if (!state.Seen[i]) + { + return VFS::VFSResult::Fail(VFS::VFSError::Corrupted); + } + } + return IsCancelled(cancelled) ? VFS::VFSResult::Fail(VFS::VFSError::Cancelled) : VFS::VFSResult::Ok(); + } + + VFS::VFSResult CookedPackageSnapshot::CaptureScene(Core::Memory::IAllocationRoute& route, size_t max_bytes, CookedObjectSnapshot* output, const std::atomic* cancelled) + { + const auto* scene = Scene(); + return scene ? CookedObjectSnapshot::Capture(Cook::CookObjectReader(m_root), scene->Result.Object, Cook::CookSceneObjectExtension, route, max_bytes, output, cancelled) : VFS::VFSResult::Fail(VFS::VFSError::InvalidPath); + } + + VFS::VFSResult CookedPackageSnapshot::CaptureAsset(const Cook::CookAssetIdentity& identity, Core::Memory::IAllocationRoute& route, size_t max_bytes, CookedObjectSnapshot* output, const std::atomic* cancelled) + { + const auto* asset = FindSelectedAsset(identity); + return asset && asset->Format ? CookedObjectSnapshot::Capture(Cook::CookObjectReader(m_root), asset->Result.Object, asset->Format->Extension, route, max_bytes, output, cancelled) : VFS::VFSResult::Fail(VFS::VFSError::NotFound); + } +} // namespace ZEngine::Managers diff --git a/ZEngine/ZEngine/Managers/CookedPackageSnapshot.h b/ZEngine/ZEngine/Managers/CookedPackageSnapshot.h new file mode 100644 index 000000000..7b0005915 --- /dev/null +++ b/ZEngine/ZEngine/Managers/CookedPackageSnapshot.h @@ -0,0 +1,66 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace ZEngine::Managers +{ + /// One selected scene's immutable parsed manifest and retained read root. + /// The supplied allocator and frozen schema registry outlive this owner and + /// its borrowers. Object bytes are captured and verified separately: a root + /// lease alone does not make files immutable. Use on one load worker unless + /// the caller provides synchronization for the rooted backend and allocator. + class CookedPackageSnapshot final + { + public: + using AssetVisitor = Core::VFS::VFSResult (*)(void*, const Cook::CookManifestAsset&); + CookedPackageSnapshot() = default; + ~CookedPackageSnapshot(); + CookedPackageSnapshot(const CookedPackageSnapshot&) = delete; + CookedPackageSnapshot& operator=(const CookedPackageSnapshot&) = delete; + CookedPackageSnapshot(CookedPackageSnapshot&& other) noexcept; + CookedPackageSnapshot& operator=(CookedPackageSnapshot&& other) noexcept; + + /// Reads a bounded private copy of manifest.json, validates its target, + /// schemas and complete typed ledger, then retains the selected closure. + /// Rebuild is rejected. Failure preserves root and output; success consumes + /// root. Limits and allocator are runtime admission policy, not a cooker run. + [[nodiscard]] static Core::VFS::VFSResult Capture(Core::VFS::VFSReadRootLease&& root, const uuids::uuid& scene, const ECS::SceneComponentSchemaRegistry& schemas, const Cook::CookLimits& limits, Cook::ICookAllocator& allocator, Cook::CookDiagnostics* diagnostics, CookedPackageSnapshot* output, const std::atomic* cancelled = nullptr); + + void Reset() noexcept; + [[nodiscard]] bool IsValid() const noexcept + { + return m_root.IsValid() && m_manifest.IsValid(); + } + [[nodiscard]] const Cook::Digest256& ManifestDigest() const noexcept + { + return m_digest; + } + [[nodiscard]] const ECS::SceneComponentSchemaRegistry* Schemas() const noexcept + { + return IsValid() ? m_schemas : nullptr; + } + [[nodiscard]] const Cook::CookManifestScene* Scene() const noexcept; + [[nodiscard]] const Cook::CookManifestAsset* FindSelectedAsset(const Cook::CookAssetIdentity& identity) const noexcept; + /// Visits each selected version once in canonical manifest order. Views + /// are borrowed until reset; visitor errors and cancellation stop the walk. + [[nodiscard]] Core::VFS::VFSResult VisitSelectedAssets(AssetVisitor visitor, void* context, const std::atomic* cancelled = nullptr) const; + [[nodiscard]] Core::Containers::ArrayView Dependencies(const Cook::CookAssetIdentity& identity) const noexcept; + /// Compare the decoded scene's complete typed root set and UUID with the + /// retained manifest before any of its references are used for rendering. + [[nodiscard]] Core::VFS::VFSResult ValidateScene(const ECS::SceneDocument& document, const ECS::SceneComponentSchemaRegistry& schemas, ECS::SceneDiagnostics* diagnostics, const std::atomic* cancelled = nullptr) const; + [[nodiscard]] Core::VFS::VFSResult CaptureScene(Core::Memory::IAllocationRoute& route, size_t max_bytes, CookedObjectSnapshot* output, const std::atomic* cancelled = nullptr); + [[nodiscard]] Core::VFS::VFSResult CaptureAsset(const Cook::CookAssetIdentity& identity, Core::Memory::IAllocationRoute& route, size_t max_bytes, CookedObjectSnapshot* output, const std::atomic* cancelled = nullptr); + + private: + Core::VFS::VFSReadRootLease m_root = {}; + Cook::CookManifest m_manifest = {}; + Cook::ICookAllocator* m_allocator = nullptr; + const ECS::SceneComponentSchemaRegistry* m_schemas = nullptr; + uint8_t* m_selected = nullptr; + uuids::uuid m_scene = {}; + Cook::Digest256 m_digest = {}; + }; +} // namespace ZEngine::Managers diff --git a/ZEngine/ZEngine/Profiling/ProfilerBuffer.cpp b/ZEngine/ZEngine/Profiling/ProfilerBuffer.cpp index e07dd6b72..4118a594e 100644 --- a/ZEngine/ZEngine/Profiling/ProfilerBuffer.cpp +++ b/ZEngine/ZEngine/Profiling/ProfilerBuffer.cpp @@ -58,6 +58,14 @@ namespace ZEngine::Profiling return count > CAPTURE_CAPACITY ? count - CAPTURE_CAPACITY : 0; } + bool ProfilerBuffer::CaptureDeadlinePassed() + { + if (!s_capture_deadline_ns) + return false; + const auto now = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); + return now >= s_capture_deadline_ns; + } + void ProfilerBuffer::Capture(const char* name, double value, bool is_value) { if (!s_capture_enabled.value.load(std::memory_order_acquire) || std::strncmp(name, s_capture_prefix, std::strlen(s_capture_prefix)) != 0) diff --git a/ZEngine/ZEngine/Profiling/ProfilerBuffer.h b/ZEngine/ZEngine/Profiling/ProfilerBuffer.h index 84199e31a..644fa1466 100644 --- a/ZEngine/ZEngine/Profiling/ProfilerBuffer.h +++ b/ZEngine/ZEngine/Profiling/ProfilerBuffer.h @@ -62,6 +62,9 @@ namespace ZEngine::Profiling static void StopCapture(); static uint32_t CaptureCount(); static uint32_t CaptureDropped(); + /// Owner-only query; true once a timed capture's deadline has passed, + /// even if no later matching sample arrived to stop recording. + static bool CaptureDeadlinePassed(); static bool WriteCaptureCsv(const char* filename); private: diff --git a/ZEngine/ZEngine/Rendering/CookedSceneGpuMailbox.cpp b/ZEngine/ZEngine/Rendering/CookedSceneGpuMailbox.cpp new file mode 100644 index 000000000..2a3d817c4 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/CookedSceneGpuMailbox.cpp @@ -0,0 +1,211 @@ +#include +#include +#include +#include +#include + +namespace ZEngine::Rendering +{ +#if ZENGINE_PROFILING + static uint64_t CancellationClockNs() noexcept + { + return static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); + } +#endif + bool CookedSceneGpuMailbox::Begin(const Managers::CookedCpuClosureCache::Lease& lease, std::span meshes, const Cook::CookAssetIdentity& environment, Core::Memory::IAllocationRoute& route, Core::Memory::TLSFSlab& metadata, RenderResourceManager& manager, Hardwares::VulkanDevice& device, RenderResourceManager::SceneResourcePreparationId scene, uint32_t max_textures, Ticket* output) + { + return TryBegin(lease, meshes, environment, route, metadata, manager, device, scene, max_textures, output) == Admission::Queued; + } + + CookedSceneGpuMailbox::Admission CookedSceneGpuMailbox::TryBegin(const Managers::CookedCpuClosureCache::Lease& lease, std::span meshes, const Cook::CookAssetIdentity& environment, Core::Memory::IAllocationRoute& route, Core::Memory::TLSFSlab& metadata, RenderResourceManager& manager, Hardwares::VulkanDevice& device, RenderResourceManager::SceneResourcePreparationId scene, uint32_t max_textures, Ticket* output, bool pause_after_mesh_record) + { + if (!output || *output || m_sealed.load(std::memory_order_acquire) || m_next_serial == std::numeric_limits::max()) + return Admission::Failed; + for (const auto& slot : m_slots) + { + const auto phase = slot.PhaseValue.load(std::memory_order_acquire); + if (phase != Phase::Free && phase != Phase::Writing && slot.Scene == scene) + return Admission::Failed; + } + for (uint32_t i = 0; i < Capacity; ++i) + { + auto& slot = m_slots[i]; + auto expected = Phase::Free; + if (!slot.PhaseValue.compare_exchange_strong(expected, Phase::Writing, std::memory_order_acq_rel)) + continue; + if (!slot.Driver.Initialize(lease, meshes, environment, route, metadata, manager, device, scene, max_textures)) + { + slot.PhaseValue.store(Phase::Free, std::memory_order_release); + return Admission::Failed; + } + slot.Serial = ++m_next_serial; + slot.Scene = scene; + slot.Result = Outcome::Pending; + slot.Cancelled.store(false, std::memory_order_relaxed); + slot.MeshRecorded.store(false, std::memory_order_relaxed); +#if ZENGINE_PROFILING + slot.CancelRequestNs.store(0, std::memory_order_relaxed); + slot.CancelObservedNs = 0; +#endif + slot.PauseAfterMeshRecord = pause_after_mesh_record; + *output = {i, slot.Serial}; + slot.PhaseValue.store(Phase::Queued, std::memory_order_release); + return Admission::Queued; + } + return Admission::Backpressure; + } + + bool CookedSceneGpuMailbox::MeshUploadRecorded(Ticket ticket) const noexcept + { + if (!ticket) + return false; + const auto& slot = m_slots[ticket.Slot]; + const auto phase = slot.PhaseValue.load(std::memory_order_acquire); + return (phase == Phase::Queued || phase == Phase::Running || phase == Phase::Finished) && slot.Serial == ticket.Serial && slot.MeshRecorded.load(std::memory_order_acquire); + } + + bool CookedSceneGpuMailbox::Cancel(Ticket ticket) noexcept + { + if (!ticket) + return false; + auto& slot = m_slots[ticket.Slot]; + const auto phase = slot.PhaseValue.load(std::memory_order_acquire); + if ((phase != Phase::Queued && phase != Phase::Running) || slot.Serial != ticket.Serial) + return false; +#if ZENGINE_PROFILING + slot.CancelRequestNs.store(CancellationClockNs(), std::memory_order_relaxed); +#endif + slot.Cancelled.store(true, std::memory_order_release); + return true; + } + + bool CookedSceneGpuMailbox::Poll(Ticket ticket, Outcome* output) const noexcept + { + if (!ticket || !output) + return false; + const auto& slot = m_slots[ticket.Slot]; + const auto phase = slot.PhaseValue.load(std::memory_order_acquire); + if ((phase != Phase::Queued && phase != Phase::Running && phase != Phase::Finished) || slot.Serial != ticket.Serial) + return false; + *output = phase == Phase::Finished ? slot.Result : Outcome::Pending; + return true; + } + + bool CookedSceneGpuMailbox::Acknowledge(Ticket ticket) noexcept + { + if (!ticket) + return false; + auto& slot = m_slots[ticket.Slot]; + if (slot.PhaseValue.load(std::memory_order_acquire) != Phase::Finished || slot.Serial != ticket.Serial) + return false; + slot.PhaseValue.store(Phase::Free, std::memory_order_release); + return true; + } + + void CookedSceneGpuMailbox::CancelAll() noexcept + { + for (auto& slot : m_slots) + { + const auto phase = slot.PhaseValue.load(std::memory_order_acquire); + if (phase == Phase::Queued || phase == Phase::Running) + slot.Cancelled.store(true, std::memory_order_release); + } + } + + bool CookedSceneGpuMailbox::DrainAfterDeviceIdle() noexcept + { + if (!m_sealed.load(std::memory_order_acquire)) + return false; + for (auto& slot : m_slots) + { + const auto phase = slot.PhaseValue.load(std::memory_order_acquire); + if (phase == Phase::Writing) + return false; + if (phase != Phase::Queued && phase != Phase::Running) + continue; + slot.Driver.Cancel(); + if (!slot.Driver.AbandonAfterDeviceIdle()) + return false; + const auto state = slot.Driver.Status(); + slot.Result = state == CookedSceneGpuPreparation::State::Failed ? Outcome::Failed : Outcome::Cancelled; + if (!slot.Driver.Reset()) + return false; + slot.PhaseValue.store(Phase::Finished, std::memory_order_release); + } + return true; + } + + bool CookedSceneGpuMailbox::PumpOne(uint8_t frame_index, uint8_t thread_index, bool cancellations_only) + { + for (uint32_t offset = 0; offset < Capacity; ++offset) + { + const uint32_t index = (m_turn + offset) % Capacity; + auto& slot = m_slots[index]; + auto phase = slot.PhaseValue.load(std::memory_order_acquire); + if (cancellations_only && !slot.Cancelled.load(std::memory_order_acquire)) + continue; + if (phase == Phase::Queued) + { + if (!slot.PhaseValue.compare_exchange_strong(phase, Phase::Running, std::memory_order_acq_rel)) + continue; + phase = Phase::Running; + } + if (phase != Phase::Running) + continue; + ZENGINE_PROFILE_SCOPE("SceneLoad.Render.CookedUploadVisit"); + m_turn = (index + 1) % Capacity; + const bool cancelled = slot.Cancelled.load(std::memory_order_acquire); + if (cancelled) + { +#if ZENGINE_PROFILING + if (!slot.CancelObservedNs) + slot.CancelObservedNs = CancellationClockNs(); +#endif + slot.Driver.Cancel(); + } + else if (slot.PauseAfterMeshRecord && slot.MeshRecorded.load(std::memory_order_acquire)) + return true; + const auto state = slot.Driver.Step(frame_index, thread_index); + if (slot.Driver.HasRecordedMeshUpload()) + slot.MeshRecorded.store(true, std::memory_order_release); + if (state == CookedSceneGpuPreparation::State::Submitted || state == CookedSceneGpuPreparation::State::Failed || state == CookedSceneGpuPreparation::State::Cancelled) + { + slot.Result = state == CookedSceneGpuPreparation::State::Submitted ? Outcome::Submitted : state == CookedSceneGpuPreparation::State::Cancelled ? Outcome::Cancelled : Outcome::Failed; + if (!slot.Driver.Reset()) + return true; +#if ZENGINE_PROFILING + if (cancelled) + { + const uint64_t request_ns = slot.CancelRequestNs.load(std::memory_order_relaxed); + const uint64_t finished_ns = CancellationClockNs(); + if (request_ns && slot.CancelObservedNs >= request_ns && finished_ns >= slot.CancelObservedNs) + ZENGINE_CORE_INFO("[CookedGpuCancelTiming] request_ns={} observed_ns={} finished_ns={}", request_ns, slot.CancelObservedNs, finished_ns) + } +#endif + slot.PhaseValue.store(Phase::Finished, std::memory_order_release); + } + return true; + } + return false; + } + + Textures::CookedTextureUpload* CookedSceneGpuMailbox::FindTexture(Ticket ticket, const Cook::CookAssetIdentity& identity) noexcept + { + if (!ticket) + return nullptr; + auto& slot = m_slots[ticket.Slot]; + const auto phase = slot.PhaseValue.load(std::memory_order_acquire); + return (phase == Phase::Queued || phase == Phase::Running) && slot.Serial == ticket.Serial ? slot.Driver.FindTexture(identity) : nullptr; + } + + bool CookedSceneGpuMailbox::Drained() const noexcept + { + for (const auto& slot : m_slots) + { + const auto phase = slot.PhaseValue.load(std::memory_order_acquire); + if (phase != Phase::Free && phase != Phase::Finished) + return false; + } + return true; + } +} // namespace ZEngine::Rendering diff --git a/ZEngine/ZEngine/Rendering/CookedSceneGpuMailbox.h b/ZEngine/ZEngine/Rendering/CookedSceneGpuMailbox.h new file mode 100644 index 000000000..f79e0d144 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/CookedSceneGpuMailbox.h @@ -0,0 +1,103 @@ +#pragma once +#include +#include + +namespace ZEngine::Rendering +{ + /// Single producer / single render consumer. Services and each admitted + /// route/slab/cache outlive the ticket until it is terminal. Seal and join + /// producers before draining on the render owner or after it has joined. + class CookedSceneGpuMailbox final + { + public: + static constexpr uint32_t Capacity = 2; + struct Ticket + { + uint32_t Slot = UINT32_MAX; + uint64_t Serial = 0; + explicit operator bool() const noexcept + { + return Slot < Capacity && Serial; + } + }; + enum class Outcome : uint8_t + { + Pending, + Submitted, + Failed, + Cancelled + }; + enum class Admission : uint8_t + { + Queued, + Backpressure, + Failed + }; + CookedSceneGpuMailbox() = default; + CookedSceneGpuMailbox(const CookedSceneGpuMailbox&) = delete; + CookedSceneGpuMailbox& operator=(const CookedSceneGpuMailbox&) = delete; + + /// Producer-only. Failure leaves the ticket, lease and scene untouched. + [[nodiscard]] bool Begin(const Managers::CookedCpuClosureCache::Lease&, std::span, const Cook::CookAssetIdentity&, Core::Memory::IAllocationRoute&, Core::Memory::TLSFSlab&, RenderResourceManager&, Hardwares::VulkanDevice&, RenderResourceManager::SceneResourcePreparationId, uint32_t max_textures, Ticket* output); + /// Distinguishes full mailbox capacity from invalid input or bounded + /// allocation/initialization failure. Failure preserves the scene. + [[nodiscard]] Admission TryBegin(const Managers::CookedCpuClosureCache::Lease&, std::span, const Cook::CookAssetIdentity&, Core::Memory::IAllocationRoute&, Core::Memory::TLSFSlab&, RenderResourceManager&, Hardwares::VulkanDevice&, RenderResourceManager::SceneResourcePreparationId, uint32_t max_textures, Ticket* output, bool pause_after_mesh_record = false); + /// Producer observation of a recorded mesh upload. The optional test + /// pause retains it until cancellation so the boundary is deterministic. + [[nodiscard]] bool MeshUploadRecorded(Ticket) const noexcept; + /// Producer-only cancellation request. A concurrent submission may win; + /// Poll supplies the terminal outcome. + [[nodiscard]] bool Cancel(Ticket) noexcept; + /// Producer-only; a submitted result can still be waiting for renderer + /// pipeline/environment readiness in the scene resource mailbox. + [[nodiscard]] bool Poll(Ticket, Outcome* output) const noexcept; + /// Producer-only; only a terminal result can be acknowledged. + [[nodiscard]] bool Acknowledge(Ticket) noexcept; + void Seal() noexcept + { + m_sealed.store(true, std::memory_order_release); + } + /// After producer threads have joined, request cancellation for all work. + void CancelAll() noexcept; + /// Joined-owner shutdown only, after sealing/joining producers and the + /// render thread, and after device idle or terminal device loss. No + /// command recorded before this call may be submitted afterwards. + [[nodiscard]] bool DrainAfterDeviceIdle() noexcept; + /// Render-owner only, one bounded operation for one request per call. + /// Cancellation-only pumping does not record new work and can run without a valid frame. + [[nodiscard]] bool PumpOne(uint8_t frame_index, uint8_t thread_index, bool cancellations_only = false); + /// Render-owner only, borrow until the next PumpOne on this mailbox. + [[nodiscard]] Textures::CookedTextureUpload* FindTexture(Ticket, const Cook::CookAssetIdentity&) noexcept; + /// Terminal results may remain unacknowledged; no driver is still live. + [[nodiscard]] bool Drained() const noexcept; + + private: + enum class Phase : uint8_t + { + Free, + Writing, + Queued, + Running, + Finished + }; + struct Slot + { + std::atomic PhaseValue{Phase::Free}; + std::atomic Cancelled{false}; + std::atomic MeshRecorded{false}; +#if ZENGINE_PROFILING + std::atomic CancelRequestNs{0}; + uint64_t CancelObservedNs = 0; // render owner only +#endif + bool PauseAfterMeshRecord = false; + uint64_t Serial = 0; + RenderResourceManager::SceneResourcePreparationId Scene = {}; + Outcome Result = Outcome::Pending; + CookedSceneGpuPreparation Driver = {}; + }; + Slot m_slots[Capacity] = {}; + std::atomic m_sealed{false}; + uint64_t m_next_serial = 0; + uint32_t m_turn = 0; + }; +} // namespace ZEngine::Rendering diff --git a/ZEngine/ZEngine/Rendering/CookedSceneGpuPreparation.cpp b/ZEngine/ZEngine/Rendering/CookedSceneGpuPreparation.cpp new file mode 100644 index 000000000..5c38514dd --- /dev/null +++ b/ZEngine/ZEngine/Rendering/CookedSceneGpuPreparation.cpp @@ -0,0 +1,163 @@ +#include +#include +#include + +namespace ZEngine::Rendering +{ + CookedSceneGpuPreparation::~CookedSceneGpuPreparation() + { + const bool reset = Reset(); + ZENGINE_VALIDATE_ASSERT(reset, "Cooked scene GPU preparation destroyed before cancellation drained") + (void) reset; + } + + bool CookedSceneGpuPreparation::Initialize(const Managers::CookedCpuClosureCache::Lease& lease, std::span meshes, const Cook::CookAssetIdentity& environment, Core::Memory::IAllocationRoute& route, Core::Memory::TLSFSlab& metadata, RenderResourceManager& manager, Hardwares::VulkanDevice& device, RenderResourceManager::SceneResourcePreparationId scene, uint32_t max_textures) + { + if (m_state != State::Empty || !scene) + { + return false; + } + RenderResourceManager::SceneEnvironmentPreparation sky = {}; + if (!manager.GetSceneEnvironmentPreparation(scene, &sky) || (sky.Input.Sky.IsHDRI() ? environment.Kind != Cook::CookAssetKind::EnvironmentMap || environment.UUID != sky.Input.Sky.EnvironmentMap : environment.IsValid())) + { + return false; + } + Managers::CookedCpuClosureCache::Lease textures = {}, mesh = {}, hdr = {}; + if (!lease.Retain(&textures) || !lease.Retain(&mesh) || (sky.Input.Sky.IsHDRI() && !lease.Retain(&hdr))) + { + return false; + } + if (!m_textures.Initialize(std::move(textures), route, manager, device, scene, max_textures)) + { + return false; + } + if (!m_meshes.Initialize(std::move(mesh), meshes, route, manager, metadata, scene)) + { + m_textures.Reset(); + return false; + } + if (sky.Input.Sky.IsHDRI() && !m_environment.Initialize(std::move(hdr), environment, manager, device, scene)) + { + (void) m_meshes.Reset(); + m_textures.Reset(); + return false; + } + m_manager = &manager; + m_scene = scene; + m_has_environment = sky.Input.Sky.IsHDRI(); + m_state = State::Pending; + return true; + } + + void CookedSceneGpuPreparation::Stop(State outcome) noexcept + { + m_after_drain = outcome; + (void) m_manager->ReleaseSceneResourcePreparation(m_scene); + m_textures.Reset(); + m_environment.Reset(); + m_meshes.Cancel(); + m_state = m_meshes.Status() == Meshes::CookedSceneMeshBatch::State::Draining ? State::Draining : outcome; + } + + void CookedSceneGpuPreparation::Cancel() noexcept + { + if (m_state == State::Pending) + Stop(State::Cancelled); + } + + CookedSceneGpuPreparation::State CookedSceneGpuPreparation::Step(uint8_t frame_index, uint8_t thread_index) + { + if (m_state == State::Draining) + { + const auto mesh = m_meshes.Step(frame_index); + if (mesh != Meshes::CookedSceneMeshBatch::State::Draining) + m_state = m_after_drain; + return m_state; + } + if (m_state != State::Pending) + return m_state; + RenderResourceManager::SceneResourcePreparationResult scene = {}; + if (!m_manager->PollSceneResourcePreparation(m_scene, &scene) || !scene.Cooked || scene.State != RenderResourceManager::SceneResourcePreparationState::Preparing) + { + Stop(State::Failed); + return m_state; + } + using TextureState = Textures::CookedSceneTextureBatch::State; + using MeshState = Meshes::CookedSceneMeshBatch::State; + using EnvironmentState = Textures::CookedSceneEnvironmentBatch::State; + for (uint32_t visited = 0; visited < 4; ++visited) + { + const uint32_t turn = m_turn++ % 4; + if (turn == 0 && m_textures.Status() != TextureState::Ready) + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Render.CookedTextures"); + if (m_textures.Step(frame_index, thread_index) == TextureState::Failed) + { + ZENGINE_CORE_ERROR("[CookedSceneGpuPreparation] Texture preparation failed") + Stop(State::Failed); + } + return m_state; + } + if (turn == 1 && m_meshes.Status() != MeshState::Ready) + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Render.CookedMeshes"); + const auto mesh = m_meshes.Step(frame_index); + if (mesh == MeshState::Failed || mesh == MeshState::Draining || mesh == MeshState::Cancelled) + Stop(State::Failed); + return m_state; + } + if (turn == 2 && m_has_environment && m_environment.Status() != EnvironmentState::Ready) + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Render.CookedEnvironment"); + if (m_environment.Step(frame_index, thread_index) == EnvironmentState::Failed) + Stop(State::Failed); + return m_state; + } + if (turn == 3 && m_textures.MaterialStatus() != TextureState::Ready && m_textures.Status() == TextureState::Ready) + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Render.CookedMaterials"); + if (m_textures.StepMaterials() == TextureState::Failed) + { + ZENGINE_CORE_ERROR("[CookedSceneGpuPreparation] Material preparation failed") + Stop(State::Failed); + } + return m_state; + } + } + if (m_textures.Status() == TextureState::Ready && m_textures.MaterialStatus() == TextureState::Ready && m_meshes.Status() == MeshState::Ready && (!m_has_environment || m_environment.Status() == EnvironmentState::Ready) && m_manager->FinishCookedSceneResources(m_scene)) + { + (void) m_meshes.Reset(); + m_textures.Reset(); + m_environment.Reset(); + m_state = State::Submitted; + } + return m_state; + } + + bool CookedSceneGpuPreparation::AbandonAfterDeviceIdle() noexcept + { + if (m_state == State::Pending) + Cancel(); + if (m_state == State::Draining) + { + if (!m_meshes.AbandonAfterDeviceIdle()) + return false; + m_state = m_after_drain; + } + return m_state != State::Pending && m_state != State::Draining; + } + + bool CookedSceneGpuPreparation::Reset() noexcept + { + if (m_state == State::Pending || m_state == State::Draining || !m_meshes.Reset()) + return false; + m_textures.Reset(); + m_environment.Reset(); + m_manager = nullptr; + m_scene = {}; + m_has_environment = false; + m_turn = 0; + m_state = State::Empty; + return true; + } +} // namespace ZEngine::Rendering diff --git a/ZEngine/ZEngine/Rendering/CookedSceneGpuPreparation.h b/ZEngine/ZEngine/Rendering/CookedSceneGpuPreparation.h new file mode 100644 index 000000000..f83875729 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/CookedSceneGpuPreparation.h @@ -0,0 +1,66 @@ +#pragma once +#include +#include +#include + +namespace ZEngine::Rendering +{ + /// Exclusive request driver. Initialize on a worker, then hand off to the + /// render owner. The caller keeps the cache, route, metadata slab, manager + /// and device alive and continues frame submissions while Step is pending. + class CookedSceneGpuPreparation final + { + public: + enum class State : uint8_t + { + Empty, + Pending, + Draining, + Submitted, + Failed, + Cancelled + }; + CookedSceneGpuPreparation() = default; + ~CookedSceneGpuPreparation(); + CookedSceneGpuPreparation(const CookedSceneGpuPreparation&) = delete; + CookedSceneGpuPreparation& operator=(const CookedSceneGpuPreparation&) = delete; + + /// The caller retains its closure lease. Failure preserves it and the + /// scene request; no GPU work or scene resource transfer has begun. + [[nodiscard]] bool Initialize(const Managers::CookedCpuClosureCache::Lease&, std::span meshes, const Cook::CookAssetIdentity& environment, Core::Memory::IAllocationRoute&, Core::Memory::TLSFSlab&, RenderResourceManager&, Hardwares::VulkanDevice&, RenderResourceManager::SceneResourcePreparationId, uint32_t max_textures); + /// Submitted means all resources were handed to the scene mailbox. + /// It is not scene/pipeline/sky readiness; poll the mailbox separately. + [[nodiscard]] State Step(uint8_t frame_index, uint8_t thread_index); + /// Cancels the scene request and releases untransferred uploads. Drain + /// accepted mesh work through Step before Reset or destruction. + void Cancel() noexcept; + /// Joined-owner shutdown only, after device idle or terminal loss and + /// after preventing all further submissions. + [[nodiscard]] bool AbandonAfterDeviceIdle() noexcept; + [[nodiscard]] bool Reset() noexcept; + [[nodiscard]] State Status() const noexcept + { + return m_state; + } + [[nodiscard]] bool HasRecordedMeshUpload() const noexcept + { + return m_meshes.HasRecordedUpload(); + } + [[nodiscard]] Textures::CookedTextureUpload* FindTexture(const Cook::CookAssetIdentity& identity) noexcept + { + return m_textures.Find(identity); + } + + private: + void Stop(State outcome) noexcept; + RenderResourceManager* m_manager = nullptr; + RenderResourceManager::SceneResourcePreparationId m_scene = {}; + Meshes::CookedSceneMeshBatch m_meshes = {}; + Textures::CookedSceneTextureBatch m_textures = {}; + Textures::CookedSceneEnvironmentBatch m_environment = {}; + bool m_has_environment = false; + uint32_t m_turn = 0; + State m_state = State::Empty; + State m_after_drain = State::Cancelled; + }; +} // namespace ZEngine::Rendering diff --git a/ZEngine/ZEngine/Rendering/Materials/CookedMaterialBinding.cpp b/ZEngine/ZEngine/Rendering/Materials/CookedMaterialBinding.cpp new file mode 100644 index 000000000..16d5a3794 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Materials/CookedMaterialBinding.cpp @@ -0,0 +1,173 @@ +#include +#include +#include +#include +#include +#include + +namespace ZEngine::Rendering::Materials +{ + CookedMaterialVersion::CookedMaterialVersion(CookedMaterialVersion&& other) noexcept : m_identity(std::exchange(other.m_identity, {})), m_digest(std::exchange(other.m_digest, {})), m_pin(std::move(other.m_pin)) {} + + CookedMaterialVersion& CookedMaterialVersion::operator=(CookedMaterialVersion&& other) noexcept + { + if (this != &other) + { + m_pin = std::move(other.m_pin); + m_identity = std::exchange(other.m_identity, {}); + m_digest = std::exchange(other.m_digest, {}); + } + return *this; + } + + bool BuildCookedMaterial(const Managers::CookedCpuAssetSet& assets, const Cook::CookAssetIdentity& identity, std::span textures, Meshes::MeshMaterial* output) + { + const auto* material = assets.FindMaterial(identity); + if (!output || !material) + { + return false; + } + for (size_t i = 0; i < textures.size(); ++i) + { + if (textures[i].Identity.Kind != Cook::CookAssetKind::Texture || !textures[i].Identity.IsValid() || !textures[i].Handle.Valid() || textures[i].Handle.Index >= INVALID_MAP_HANDLE || (i && !(textures[i - 1].Identity < textures[i].Identity))) + { + return false; + } + } + const auto vec = [](const Core::Containers::FixedArray& value) { return Core::Maths::Vec4f{value[0], value[1], value[2], value[3]}; }; + Meshes::MeshMaterial candidate = {}; + candidate.AmbientColor = vec(material->AmbientColor); + candidate.AlbedoColor = vec(material->AlbedoColor); + candidate.EmissiveColor = vec(material->EmissiveColor); + candidate.RoughnessColor = vec(material->RoughnessColor); + candidate.SpecularColor = vec(material->SpecularColor); + candidate.Factors = vec(material->Factors); + candidate.AlphaMode = material->AlphaMode; + uint32_t* maps[] = {&candidate.AlbedoMap, &candidate.EmissiveMap, &candidate.NormalMap, &candidate.OpacityMap, &candidate.SpecularMap}; + static_assert(std::size(maps) == static_cast(Importers::AssetCodec::MaterialTextureSlot::Count)); + for (size_t i = 0; i < material->Textures.size(); ++i) + { + const auto& texture = material->Textures[i]; + if (texture.UUID.is_nil()) + { + continue; + } + const Cook::CookAssetIdentity expected = {Cook::CookAssetKind::Texture, texture.UUID}; + const auto* version = assets.FindVersion(expected); + if (!version || !assets.FindTexture(expected) || textures.empty()) + { + return false; + } + const auto binding = std::lower_bound(textures.begin(), textures.end(), expected, [](const CookedTextureBinding& value, const Cook::CookAssetIdentity& key) { return value.Identity < key; }); + if (binding == textures.end() || binding->Identity != expected || binding->ObjectDigest != version->ObjectDigest) + { + return false; + } + *maps[i] = static_cast(binding->Handle.Index); + } + *output = candidate; + return true; + } + + SceneTextureDescriptorState PrepareCookedMaterial(const Managers::CookedCpuAssetSet& assets, const Cook::CookAssetIdentity& identity, std::span uploads, RenderResourceManager& manager, CookedMaterialVersion* output) + { + const auto* version = assets.FindVersion(identity); + if (!output || *output || !version || identity.Kind != Cook::CookAssetKind::Material) + { + return SceneTextureDescriptorState::Failed; + } + RenderResourceManager::MaterialVersionPin pin = {}; + const auto state = PrepareCookedMaterial(assets, identity, uploads, manager, &pin); + if (state == SceneTextureDescriptorState::Ready) + { + output->m_identity = identity; + output->m_digest = version->ObjectDigest; + output->m_pin = std::move(pin); + } + return state; + } + + SceneTextureDescriptorState PrepareCookedMaterial(const Managers::CookedCpuAssetSet& assets, const Cook::CookAssetIdentity& identity, std::span uploads, RenderResourceManager& manager, RenderResourceManager::MaterialVersionPin* output) + { + const auto* material = assets.FindMaterial(identity); + if (!output || *output || !material || uploads.size() > 5) + { + return SceneTextureDescriptorState::Failed; + } + std::array used = {}; + std::array map_uploads = {}; + map_uploads.fill(UINT32_MAX); + for (size_t i = 0; i < uploads.size(); ++i) + { + if (!uploads[i] || uploads[i]->Identity().Kind != Cook::CookAssetKind::Texture || !uploads[i]->Identity().IsValid() || (i && !(uploads[i - 1]->Identity() < uploads[i]->Identity()))) + { + return SceneTextureDescriptorState::Failed; + } + const auto* version = assets.FindVersion(uploads[i]->Identity()); + if (!version || !assets.FindTexture(uploads[i]->Identity()) || version->ObjectDigest != uploads[i]->Digest()) + { + return SceneTextureDescriptorState::Failed; + } + } + for (size_t map = 0; map < material->Textures.size(); ++map) + { + const auto& uuid = material->Textures[map].UUID; + if (uuid.is_nil()) + { + continue; + } + const Cook::CookAssetIdentity expected = {Cook::CookAssetKind::Texture, uuid}; + const auto it = std::lower_bound(uploads.begin(), uploads.end(), expected, [](const auto* upload, const auto& key) { return upload->Identity() < key; }); + if (it == uploads.end() || (*it)->Identity() != expected) + { + return SceneTextureDescriptorState::Failed; + } + const auto index = static_cast(it - uploads.begin()); + used[index] = true; + map_uploads[map] = index; + } + std::array bindings = {}; + std::array pins = {}; + for (size_t i = 0; i < uploads.size(); ++i) + { + if (!used[i]) + { + return SceneTextureDescriptorState::Failed; + } + const auto state = uploads[i]->Poll(); + if (state == Textures::CookedTextureUpload::Readiness::Failed) + { + return SceneTextureDescriptorState::Failed; + } + if (state != Textures::CookedTextureUpload::Readiness::Ready) + { + return SceneTextureDescriptorState::Pending; + } + const auto* binding = uploads[i]->ReadyBinding(); + if (!binding) + { + return SceneTextureDescriptorState::Pending; + } + bindings[i] = *binding; + pins[i] = uploads[i]->RetainPreparedVersion(); + if (!pins[i]) + { + return SceneTextureDescriptorState::Failed; + } + } + Meshes::MeshMaterial gpu_material = {}; + if (!BuildCookedMaterial(assets, identity, {bindings.data(), uploads.size()}, &gpu_material)) + { + return SceneTextureDescriptorState::Failed; + } + const RenderResourceManager::TextureVersionPin* map_pins[5] = {}; + for (size_t map = 0; map < map_uploads.size(); ++map) + { + if (map_uploads[map] != UINT32_MAX) + { + map_pins[map] = &pins[map_uploads[map]]; + } + } + return manager.PrepareExplicitMaterialVersion(gpu_material, map_pins, output); + } +} // namespace ZEngine::Rendering::Materials diff --git a/ZEngine/ZEngine/Rendering/Materials/CookedMaterialBinding.h b/ZEngine/ZEngine/Rendering/Materials/CookedMaterialBinding.h new file mode 100644 index 000000000..62d9013c7 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Materials/CookedMaterialBinding.h @@ -0,0 +1,78 @@ +#pragma once +#include +#include +#include +#include + +namespace ZEngine::Rendering::Textures +{ + class CookedTextureUpload; +} + +namespace ZEngine::Rendering::Materials +{ + /// An upload owner associates its exact cooked content with a GPU handle. + /// This is only a binding declaration: the renderer must separately retain + /// that handle's generation and prove upload and descriptor readiness. + struct CookedTextureBinding + { + Cook::CookAssetIdentity Identity = {}; + Cook::Digest256 ObjectDigest = {}; + Textures::TextureHandle Handle = {}; + }; + + /// Exact cooked material identity and the retained shader/texture version. + /// Render-owner only; its resource manager outlives this object. + class CookedMaterialVersion final + { + public: + CookedMaterialVersion() = default; + CookedMaterialVersion(const CookedMaterialVersion&) = delete; + CookedMaterialVersion& operator=(const CookedMaterialVersion&) = delete; + CookedMaterialVersion(CookedMaterialVersion&&) noexcept; + CookedMaterialVersion& operator=(CookedMaterialVersion&&) noexcept; + explicit operator bool() const + { + return bool(m_pin); + } + const Cook::CookAssetIdentity& Identity() const noexcept + { + return m_identity; + } + const Cook::Digest256& Digest() const noexcept + { + return m_digest; + } + const RenderResourceManager::MaterialVersionPin& Pin() const noexcept + { + return m_pin; + } + void Reset() + { + m_pin.Reset(); + m_identity = {}; + m_digest = {}; + } + + private: + friend SceneTextureDescriptorState PrepareCookedMaterial(const Managers::CookedCpuAssetSet&, const Cook::CookAssetIdentity&, std::span, RenderResourceManager&, CookedMaterialVersion*); + Cook::CookAssetIdentity m_identity = {}; + Cook::Digest256 m_digest = {}; + RenderResourceManager::MaterialVersionPin m_pin = {}; + }; + + /// Build shader values from the selected immutable material and explicitly + /// supplied texture versions. Bindings are sorted and unique by typed + /// identity. Required maps never use a fallback; nil maps use the shader's + /// defined sentinel. Failure leaves output unchanged. + [[nodiscard]] bool BuildCookedMaterial(const Managers::CookedCpuAssetSet&, const Cook::CookAssetIdentity&, std::span, Meshes::MeshMaterial* output); + + /// Prepare a retained material directly from the verified CPU closure and + /// its exact uploaded texture transactions. Inputs are sorted, unique and + /// limited to the material's required maps. Pending/Failed preserve output. + /// Ready retains the exact texture generations and descriptor publications; + /// the assets and upload transactions may then retire independently. The + /// render owner calls this after arranging graphics-family acquire work. + [[nodiscard]] SceneTextureDescriptorState PrepareCookedMaterial(const Managers::CookedCpuAssetSet&, const Cook::CookAssetIdentity&, std::span, RenderResourceManager&, RenderResourceManager::MaterialVersionPin* output); + [[nodiscard]] SceneTextureDescriptorState PrepareCookedMaterial(const Managers::CookedCpuAssetSet&, const Cook::CookAssetIdentity&, std::span, RenderResourceManager&, CookedMaterialVersion* output); +} // namespace ZEngine::Rendering::Materials diff --git a/ZEngine/ZEngine/Rendering/Meshes/CookedMeshStaging.cpp b/ZEngine/ZEngine/Rendering/Meshes/CookedMeshStaging.cpp new file mode 100644 index 000000000..5468b30f7 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Meshes/CookedMeshStaging.cpp @@ -0,0 +1,100 @@ +#include +#include + +namespace ZEngine::Rendering::Meshes +{ + bool CookedMeshStaging::Capture(Managers::CookedCpuClosureCache::Lease&& lease, const Cook::CookAssetIdentity& identity) + { + Managers::CookedCpuClosureCache::Result result = {}; + if (m_phase != Phase::Empty || identity.Kind != Cook::CookAssetKind::Mesh || !identity.IsValid() || !lease.Poll(&result) || result.StateValue != Managers::CookedCpuClosureCache::State::Ready || !result.Assets) + { + return false; + } + const auto* version = result.Assets->FindVersion(identity); + const auto* mesh = result.Assets->FindMesh(identity); + constexpr size_t floats_per_vertex = 8; + if (!version || !mesh || mesh->MeshUUID != identity.UUID || mesh->Vertices.empty() || mesh->Indices.empty() || mesh->Vertices.size() % floats_per_vertex != 0 || mesh->Vertices.size() / floats_per_vertex > UINT32_MAX || mesh->Indices.size() > UINT32_MAX || mesh->Vertices.size() > SIZE_MAX / sizeof(float) || mesh->Indices.size() > SIZE_MAX / sizeof(uint32_t)) + { + return false; + } + const size_t vertex_bytes = mesh->Vertices.size() * sizeof(float); + const size_t index_bytes = mesh->Indices.size() * sizeof(uint32_t); + if (vertex_bytes > SIZE_MAX - index_bytes) + { + return false; + } + m_identity = identity; + m_digest = version->ObjectDigest; + m_mesh = mesh; + m_vertex_bytes = vertex_bytes; + m_index_bytes = index_bytes; + m_lease = std::move(lease); + m_phase = Phase::SourceReady; + return true; + } + + bool CookedMeshStaging::Prepare(RenderResourceManager& manager, VkResult* status) + { + if (m_phase != Phase::SourceReady) + { + if (status) + { + *status = VK_ERROR_INITIALIZATION_FAILED; + } + return false; + } + if (!manager.PrepareUploadStaging(m_vertex_bytes + m_index_bytes, &m_staging, status)) + { + return false; + } + m_phase = Phase::Staging; + return true; + } + + bool CookedMeshStaging::StageOne(VkResult* status) + { + if (m_phase != Phase::Staging || !m_mesh) + { + if (status) + { + *status = VK_ERROR_INITIALIZATION_FAILED; + } + return false; + } + const size_t offset = static_cast(m_staging.Written()); + const bool vertices = offset < m_vertex_bytes; + const size_t source_offset = vertices ? offset : offset - m_vertex_bytes; + const size_t source_size = vertices ? m_vertex_bytes : m_index_bytes; + if (source_offset >= source_size) + { + if (status) + { + *status = VK_ERROR_INITIALIZATION_FAILED; + } + return false; + } + const auto* data = vertices ? reinterpret_cast(m_mesh->Vertices.data()) : reinterpret_cast(m_mesh->Indices.data()); + size_t consumed = 0; + if (!m_staging.Append({data + source_offset, source_size - source_offset}, &consumed, status)) + { + return false; + } + if (m_staging.Ready()) + { + m_phase = Phase::Staged; + } + return true; + } + + void CookedMeshStaging::Reset() noexcept + { + m_staging.Reset(); + m_lease.Reset(); + m_mesh = nullptr; + m_identity = {}; + m_digest = {}; + m_vertex_bytes = 0; + m_index_bytes = 0; + m_phase = Phase::Empty; + } +} // namespace ZEngine::Rendering::Meshes diff --git a/ZEngine/ZEngine/Rendering/Meshes/CookedMeshStaging.h b/ZEngine/ZEngine/Rendering/Meshes/CookedMeshStaging.h new file mode 100644 index 000000000..1ecfda148 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Meshes/CookedMeshStaging.h @@ -0,0 +1,79 @@ +#pragma once +#include +#include + +namespace ZEngine::Rendering::Meshes +{ + /// One immutable cooked mesh and its unpublished host staging allocation. + /// The closure lease retains the exact decoded geometry and metadata. This + /// object is exclusive; synchronize handoffs between its worker and render + /// owner. The cache, resource manager and device outlive it. + class CookedMeshStaging final + { + public: + enum class Phase : uint8_t + { + Empty, + SourceReady, + Staging, + Staged + }; + + CookedMeshStaging() = default; + ~CookedMeshStaging() = default; + CookedMeshStaging(const CookedMeshStaging&) = delete; + CookedMeshStaging& operator=(const CookedMeshStaging&) = delete; + + /// Consumes the ready closure lease only after validating its identity + /// and decoded layout. No source asset or registry is consulted. + [[nodiscard]] bool Capture(Managers::CookedCpuClosureCache::Lease&& lease, const Cook::CookAssetIdentity& identity); + /// Render-owner allocation. Failure leaves the captured source intact. + [[nodiscard]] bool Prepare(RenderResourceManager& manager, VkResult* status = nullptr); + /// Copies at most one staging slice; vertices precede indices. Failure + /// preserves the cursor. No geometry upload is submitted by this call. + [[nodiscard]] bool StageOne(VkResult* status = nullptr); + void Reset() noexcept; + + [[nodiscard]] Phase State() const noexcept + { + return m_phase; + } + [[nodiscard]] VkDeviceSize StagedBytes() const noexcept + { + return m_staging.Written(); + } + [[nodiscard]] size_t VertexBytes() const noexcept + { + return m_vertex_bytes; + } + [[nodiscard]] size_t IndexBytes() const noexcept + { + return m_index_bytes; + } + [[nodiscard]] const Cook::CookAssetIdentity& Identity() const noexcept + { + return m_identity; + } + [[nodiscard]] const Cook::Digest256& Digest() const noexcept + { + return m_digest; + } + /// Borrowed only while this staging owner remains alive and unchanged. + [[nodiscard]] const Importers::AssetMesh* Mesh() const noexcept + { + return m_mesh; + } + + private: + friend class ZEngine::Rendering::RenderResourceManager; + friend struct ::RRMUploadBatchTestHelper; + Managers::CookedCpuClosureCache::Lease m_lease = {}; + const Importers::AssetMesh* m_mesh = nullptr; + RenderResourceManager::UploadStaging m_staging = {}; + Cook::CookAssetIdentity m_identity = {}; + Cook::Digest256 m_digest = {}; + size_t m_vertex_bytes = 0; + size_t m_index_bytes = 0; + Phase m_phase = Phase::Empty; + }; +} // namespace ZEngine::Rendering::Meshes diff --git a/ZEngine/ZEngine/Rendering/Meshes/CookedSceneMeshBatch.cpp b/ZEngine/ZEngine/Rendering/Meshes/CookedSceneMeshBatch.cpp new file mode 100644 index 000000000..a58c05752 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Meshes/CookedSceneMeshBatch.cpp @@ -0,0 +1,194 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace ZEngine::Rendering::Meshes +{ + CookedSceneMeshBatch::~CookedSceneMeshBatch() + { + const bool reset = Reset(); + ZENGINE_VALIDATE_ASSERT(reset, "Cooked mesh batch destroyed before accepted uploads drained") + (void) reset; + } + + bool CookedSceneMeshBatch::Initialize(Managers::CookedCpuClosureCache::Lease&& lease, std::span meshes, Core::Memory::IAllocationRoute& route, RenderResourceManager& manager, Core::Memory::TLSFSlab& metadata, RenderResourceManager::SceneResourcePreparationId scene) + { + Managers::CookedCpuClosureCache::Result result = {}; + RenderResourceManager::SceneResourcePreparationResult preparation = {}; + if (m_state != State::Empty || !scene || meshes.size() > RenderResourceManager::SceneGeometryMeshCapacity || !lease.Poll(&result) || result.StateValue != Managers::CookedCpuClosureCache::State::Ready || !result.Assets || !manager.PollSceneResourcePreparation(scene, &preparation) || !preparation.Cooked || preparation.CookedManifest != result.Assets->ManifestDigest() || preparation.CookedScene != result.Assets->SceneUUID() || preparation.State != RenderResourceManager::SceneResourcePreparationState::Preparing) + { + return false; + } + for (size_t i = 0; i < meshes.size(); ++i) + { + const Cook::CookAssetIdentity identity = {Cook::CookAssetKind::Mesh, meshes[i]}; + if (!identity.IsValid() || (i && !(meshes[i - 1] < meshes[i])) || !result.Assets->FindMesh(identity)) + { + return false; + } + } + static_assert(alignof(uuids::uuid) <= alignof(std::max_align_t)); + if (meshes.size() > std::numeric_limits::max() / sizeof(uuids::uuid)) + return false; + auto* copy = meshes.empty() ? nullptr : static_cast(route.Allocate(meshes.size() * sizeof(uuids::uuid))); + if (!meshes.empty() && !copy) + return false; + if (copy) + { + std::uninitialized_copy(meshes.begin(), meshes.end(), copy); + } + m_lease = std::move(lease); + m_route = &route; + m_manager = &manager; + m_metadata = &metadata; + m_scene = scene; + m_meshes = copy; + m_count = static_cast(meshes.size()); + m_state = m_count ? State::Pending : State::Ready; + return true; + } + + void CookedSceneMeshBatch::Fail() noexcept + { + m_after_drain = State::Failed; + if (m_upload.Serial) + { + (void) m_manager->CancelCookedMeshUpload(m_upload); + m_state = State::Draining; + } + else + { + m_state = State::Failed; + } + } + + void CookedSceneMeshBatch::Cancel() noexcept + { + if (m_state != State::Pending) + return; + m_after_drain = State::Cancelled; + if (m_upload.Serial) + { + (void) m_manager->CancelCookedMeshUpload(m_upload); + m_state = State::Draining; + } + else + { + m_state = State::Cancelled; + } + } + + CookedSceneMeshBatch::State CookedSceneMeshBatch::Step(uint8_t frame_index) + { + if (m_state == State::Draining) + { + const auto result = m_manager->PollCookedMeshUpload(m_upload); + if (!result.CleanupPending && m_manager->ReleaseCookedMeshUpload(m_upload)) + { + m_upload = {}; + m_state = m_after_drain; + } + return m_state; + } + if (m_state != State::Pending) + return m_state; + Managers::CookedCpuClosureCache::Result assets = {}; + RenderResourceManager::SceneResourcePreparationResult scene = {}; + if (!m_lease.Poll(&assets) || assets.StateValue != Managers::CookedCpuClosureCache::State::Ready || !assets.Assets || !m_manager->PollSceneResourcePreparation(m_scene, &scene) || !scene.Cooked || scene.State != RenderResourceManager::SceneResourcePreparationState::Preparing) + { + Fail(); + return m_state; + } + if (m_upload.Serial) + { + const auto result = m_manager->PollCookedMeshUpload(m_upload); + if (result.State == RenderResourceManager::CookedMeshUploadState::Pending) + return m_state; + if (result.State != RenderResourceManager::CookedMeshUploadState::Ready) + { + Fail(); + return m_state; + } + RenderResourceManager::CookedMeshVersion version = {}; + if (!m_manager->TakeCookedMeshUpload(m_upload, &version)) + { + Fail(); + return m_state; + } + m_upload = {}; + if (!m_manager->AppendCookedSceneMesh(m_scene, *assets.Assets, version)) + { + Fail(); + return m_state; + } + if (++m_cursor == m_count) + m_state = State::Ready; + return m_state; + } + switch (m_staging.State()) + { + case CookedMeshStaging::Phase::Empty: + { + Managers::CookedCpuClosureCache::Lease retained = {}; + const Cook::CookAssetIdentity identity = {Cook::CookAssetKind::Mesh, m_meshes[m_cursor]}; + if (!m_lease.Retain(&retained) || !m_staging.Capture(std::move(retained), identity) || !m_staging.Prepare(*m_manager)) + Fail(); + break; + } + case CookedMeshStaging::Phase::Staging: + if (!m_staging.StageOne()) + Fail(); + break; + case CookedMeshStaging::Phase::Staged: + { + const auto admission = m_manager->RecordCookedMeshUpload(frame_index, m_staging, *m_metadata, &m_upload); + if (admission == RenderResourceManager::CookedMeshAdmission::Failed) + Fail(); + break; + } + default: + Fail(); + break; + } + return m_state; + } + + bool CookedSceneMeshBatch::AbandonAfterDeviceIdle() noexcept + { + if (m_upload.Serial && !m_manager->AbandonCookedMeshUploadAfterDeviceIdle(m_upload)) + { + return false; + } + m_upload = {}; + if (m_state == State::Draining) + { + m_state = m_after_drain; + } + return true; + } + + bool CookedSceneMeshBatch::Reset() noexcept + { + if (m_upload.Serial) + return false; + m_staging.Reset(); + if (m_meshes) + { + std::destroy_n(m_meshes, m_count); + m_route->Free(m_meshes); + } + m_lease.Reset(); + m_route = nullptr; + m_manager = nullptr; + m_metadata = nullptr; + m_scene = {}; + m_meshes = nullptr; + m_count = m_cursor = 0; + m_state = State::Empty; + return true; + } +} // namespace ZEngine::Rendering::Meshes diff --git a/ZEngine/ZEngine/Rendering/Meshes/CookedSceneMeshBatch.h b/ZEngine/ZEngine/Rendering/Meshes/CookedSceneMeshBatch.h new file mode 100644 index 000000000..9cc3227e9 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Meshes/CookedSceneMeshBatch.h @@ -0,0 +1,63 @@ +#pragma once +#include + +namespace ZEngine::Rendering::Meshes +{ + /// Exclusive scene mesh upload transaction. Initialize on a worker, then + /// transfer to the render owner. Each Step performs one bounded staging + /// slice, admission or completion poll. Cache, route, manager and metadata + /// slab outlive this owner and all resulting scene pins. + class CookedSceneMeshBatch final + { + public: + enum class State : uint8_t + { + Empty, + Pending, + Draining, + Ready, + Failed, + Cancelled + }; + CookedSceneMeshBatch() = default; + ~CookedSceneMeshBatch(); + CookedSceneMeshBatch(const CookedSceneMeshBatch&) = delete; + CookedSceneMeshBatch& operator=(const CookedSceneMeshBatch&) = delete; + + /// Copies sorted unique mesh identities using the caller's route. A + /// refusal preserves the lease and leaves the batch empty. + [[nodiscard]] bool Initialize(Managers::CookedCpuClosureCache::Lease&&, std::span, Core::Memory::IAllocationRoute&, RenderResourceManager&, Core::Memory::TLSFSlab&, RenderResourceManager::SceneResourcePreparationId); + [[nodiscard]] State Step(uint8_t frame_index); + /// Cancellation can be observed before accepted GPU work is retired. + /// Keep calling Step until Cancelled/Failed before resetting the owner. + void Cancel() noexcept; + /// Joined-owner shutdown only, after device idle or terminal loss and + /// after preventing all further submissions. + [[nodiscard]] bool AbandonAfterDeviceIdle() noexcept; + [[nodiscard]] bool Reset() noexcept; + [[nodiscard]] State Status() const noexcept + { + return m_state; + } + /// Render-owner observation; the recorded upload still owns its staging. + [[nodiscard]] bool HasRecordedUpload() const noexcept + { + return m_upload.Serial != 0; + } + + private: + void Fail() noexcept; + Managers::CookedCpuClosureCache::Lease m_lease = {}; + Core::Memory::IAllocationRoute* m_route = nullptr; + RenderResourceManager* m_manager = nullptr; + Core::Memory::TLSFSlab* m_metadata = nullptr; + RenderResourceManager::SceneResourcePreparationId m_scene = {}; + uuids::uuid* m_meshes = nullptr; + uint32_t m_count = 0; + uint32_t m_cursor = 0; + CookedMeshStaging m_staging = {}; + RenderResourceManager::CookedMeshUploadId m_upload = {}; + State m_state = State::Empty; + State m_after_drain = State::Cancelled; + }; +} // namespace ZEngine::Rendering::Meshes diff --git a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp index 9f6a36345..909784942 100644 --- a/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp +++ b/ZEngine/ZEngine/Rendering/RenderResourceManager.cpp @@ -15,9 +15,13 @@ #include #include #include +#include +#include #include #include #include +#include +#include #include #include #include @@ -33,6 +37,7 @@ #include #include #include +#include using namespace ZEngine::Core::Memory; using namespace ZEngine::Core::VFS; @@ -40,6 +45,10 @@ using namespace ZEngine::Hardwares; using namespace ZEngine::Importers; using namespace ZEngine::Managers; using namespace ZEngine::Helpers; +using ZEngine::Rendering::Primitives::ImageMemoryBarrier; +using ZEngine::Rendering::Specifications::ImageFormat; +using ZEngine::Rendering::Specifications::ImageLayout; +using ZEngine::Rendering::Specifications::ImageMemoryBarrierSpecification; namespace ZEngine::Rendering { @@ -482,6 +491,7 @@ namespace ZEngine::Rendering // The render thread has joined; submitted consumers are idle (or the // device is terminally lost), and no recorded frame can be submitted later. ReleaseSceneAndFrameResourcesAfterIdle(); + ReleaseCookedMeshUploadsAfterIdle(); ZENGINE_VALIDATE_ASSERT(m_pinned_mesh_geometry_count == 0, "RenderResourceManager::Shutdown: explicit geometry pins must retire before their resource owner") ZENGINE_VALIDATE_ASSERT(m_texture_version_pin_count == 0, "RenderResourceManager::Shutdown: material/texture pins must retire before their resource owner") for (uint32_t i = 0; i < m_mesh_slot_count; ++i) @@ -531,6 +541,16 @@ namespace ZEngine::Rendering m_upload_cmd_mgr->Deinitialize(); m_upload_cmd_mgr = nullptr; } + // Includes unsubmitted or rejected recordings, which have no timeline + // signal. Their command buffers are gone and accepted work is idle. + for (auto& frame : m_batch_frames) + { + for (uint32_t i = 0; i < frame.StagingCount; ++i) + { + m_device->GpuMem.FreeBuffer(frame.StagingBuffers[i]); + } + frame.StagingCount = 0; + } m_async_uploads.Deinitialize(); if (m_sync_upload_fence) { @@ -587,6 +607,7 @@ namespace ZEngine::Rendering } m_active_frame_index = static_cast(frame_index); RetireBatchStagings(); + PollCookedMeshUploads(); DispatchQueuedTextureDecodes(); FlushPendingMeshReleases(); m_streaming_mgr.Tick(frame_index); @@ -607,6 +628,7 @@ namespace ZEngine::Rendering { EndBatchUpload(); } + PollCookedMeshUploads(); PollSceneResourcePreparations(); } @@ -666,12 +688,13 @@ namespace ZEngine::Rendering VkBuffer DstBuffer; VkDeviceSize DstOffset; size_t ByteSize; + VkDeviceSize SrcOffset = 0; }; static void RecordGlobalBufferCopy(VkCommandBuffer cmd, void* ctx_ptr) { auto* ctx = static_cast(ctx_ptr); - VkBufferCopy region{.srcOffset = 0, .dstOffset = ctx->DstOffset, .size = ctx->ByteSize}; + VkBufferCopy region{.srcOffset = ctx->SrcOffset, .dstOffset = ctx->DstOffset, .size = ctx->ByteSize}; vkCmdCopyBuffer(cmd, ctx->SrcBuffer, ctx->DstBuffer, 1, ®ion); VkBufferMemoryBarrier barrier{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; @@ -1048,6 +1071,68 @@ namespace ZEngine::Rendering } } + RenderResourceManager::CookedMeshAdmission RenderResourceManager::TryEnsureCookedBatchOpen(uint8_t frame_index) + { + if (!m_device || !m_device->LogicalDevice || !m_upload_cmd_mgr || !m_batch_timeline || frame_index >= m_batch_frames.size() || m_batch_submission_failed || m_device->IsDeviceLost.load(std::memory_order_acquire) || m_batch_next_value == UINT64_MAX) + { + return CookedMeshAdmission::Failed; + } + if (m_batch_mode) + { + return m_batch_frame_index == frame_index ? CookedMeshAdmission::Recorded : CookedMeshAdmission::Retry; + } + auto& frame = m_batch_frames[frame_index]; + if (frame.LastSignal) + { + uint64_t completed = 0; + const VkResult result = vkGetSemaphoreCounterValue(m_device->LogicalDevice, m_batch_timeline->GetHandle(), &completed); + if (result != VK_SUCCESS) + { + m_device->CheckDeviceLost(result, "Cooked mesh batch completion query"); + return CookedMeshAdmission::Failed; + } + if (completed < frame.LastSignal) + { + return CookedMeshAdmission::Retry; + } + } + // BeginFrame retires the preceding batch's staging. Do not replace it + // or perform an unbounded staging cleanup inside this admission call. + if (frame.StagingCount) + { + return CookedMeshAdmission::Retry; + } + const size_t stride = size_t(m_upload_cmd_mgr->MaxBufferPerPool) * m_upload_cmd_mgr->MaxBufferPerPool; + const size_t index = size_t(frame_index) * m_upload_cmd_mgr->TotalThreadCount * stride; + if (!stride || !m_upload_cmd_mgr->TotalThreadCount || index >= m_upload_cmd_mgr->InstantGraphicsCommandBuffers.size()) + { + return CookedMeshAdmission::Failed; + } + auto* command = m_upload_cmd_mgr->InstantGraphicsCommandBuffers[index]; + if (!command || !command->GetHandle() || command->BufferType != Hardwares::CommandBufferType::Primary) + { + return CookedMeshAdmission::Failed; + } + VkResult result = vkResetCommandBuffer(command->GetHandle(), 0); + if (result == VK_SUCCESS) + { + command->ResetState(); + VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; + result = vkBeginCommandBuffer(command->GetHandle(), &begin); + } + if (result != VK_SUCCESS) + { + m_device->CheckDeviceLost(result, "Cooked mesh batch recording"); + return CookedMeshAdmission::Failed; + } + command->SetState(Hardwares::CommandBufferState::Recording); + m_batch_frame_index = frame_index; + m_batch_cmd = command; + m_batch_staging_base_count = frame.StagingCount; + m_batch_mode = true; + return CookedMeshAdmission::Recorded; + } + void RenderResourceManager::BeginBatchUpload(uint8_t frame_index) { m_batch_frame_index = frame_index; @@ -1071,19 +1156,47 @@ namespace ZEngine::Rendering m_batch_cmd->ResetState(); vkResetCommandBuffer(m_batch_cmd->GetHandle(), 0); m_batch_cmd->Begin(); - m_batch_mode = true; + m_batch_staging_base_count = frame.StagingCount; + m_batch_mode = true; + } + + void RenderResourceManager::DiscardUnsubmittedBatchStaging() + { + auto& frame = m_batch_frames[m_batch_frame_index]; + ZENGINE_VALIDATE_ASSERT(m_batch_staging_base_count <= frame.StagingCount, "Invalid geometry batch staging boundary") + while (frame.StagingCount > m_batch_staging_base_count) + { + auto& staging = frame.StagingBuffers[--frame.StagingCount]; + m_device->GpuMem.FreeBuffer(staging); + staging = {}; + } } void RenderResourceManager::EndBatchUpload() { - m_batch_cmd->End(); + const VkResult end_result = vkEndCommandBuffer(m_batch_cmd->GetHandle()); + if (end_result != VK_SUCCESS) + { + m_device->CheckDeviceLost(end_result, "Geometry batch recording end"); + m_batch_submission_failed = true; + m_batch_mode = false; + m_batch_cmd = nullptr; + if (!m_device->IsDeviceLost.load(std::memory_order_acquire)) + DiscardUnsubmittedBatchStaging(); + // Fail unaccepted records before any unrelated later batch can + // advance the timeline past the value they expected. + PollCookedMeshUploads(); + return; + } + m_batch_cmd->SetState(Hardwares::CommandBufferState::Executable); // Submit without blocking so a mesh drop never stalls the render thread. Present() // consumes the resulting timeline operation in the graphics submission below. // Signals m_batch_timeline — a dedicated semaphore with exactly one writer (this // function) — rather than DeviceSwapchain::RenderTimeline, which Present() also // drives independently. - uint64_t signal_value = ++m_batch_next_value; + const uint64_t previous_signal = m_batch_frames[m_batch_frame_index].LastSignal; + uint64_t signal_value = ++m_batch_next_value; m_batch_frames[m_batch_frame_index].LastSignal = signal_value; // The global geometry buffers are bound as whole-buffer storage descriptors. @@ -1125,7 +1238,21 @@ namespace ZEngine::Rendering }; } - if (m_device->QueueSubmit(m_batch_cmd, m_batch_timeline, signal_value, copy_waits, copy_wait_count)) + bool injected_rejection = false; + if (m_reject_cooked_batch_probe.load(std::memory_order_acquire)) + { + for (const auto& upload : m_cooked_mesh_uploads) + { + if (upload.State == CookedMeshUploadState::Pending && upload.SignalValue == signal_value) + { + injected_rejection = m_reject_cooked_batch_probe.exchange(false, std::memory_order_acq_rel); + break; + } + } + } + if (injected_rejection) + ZENGINE_CORE_INFO("[CookedUploadProbe] Rejected recorded cooked batch before queue submission") + if (!injected_rejection && m_device->QueueSubmit(m_batch_cmd, m_batch_timeline, signal_value, copy_waits, copy_wait_count)) { m_batch_accepted_value = signal_value; m_device->EnqueueAsyncGPUOperation({consumer_wait_stage, signal_value, m_batch_timeline}); @@ -1135,7 +1262,7 @@ namespace ZEngine::Rendering // A rejected submit cannot supply a completion gate. Device loss is // terminal; never let a later counter value turn it into readiness. m_batch_submission_failed = true; - m_batch_frames[m_batch_frame_index].LastSignal = 0; + m_batch_frames[m_batch_frame_index].LastSignal = previous_signal; } // Left in m_batch_frames[m_batch_frame_index] for RetireBatchStagings (or the next @@ -1143,6 +1270,12 @@ namespace ZEngine::Rendering // this copy has completed. m_batch_mode = false; m_batch_cmd = nullptr; + if (m_batch_submission_failed) + { + if (!m_device->IsDeviceLost.load(std::memory_order_acquire)) + DiscardUnsubmittedBatchStaging(); + PollCookedMeshUploads(); + } } void RenderResourceManager::AppendToGlobalBuffer(BufferView& dst_buf, const void* data, size_t byte_size, VkDeviceSize byte_offset, uint32_t frame_index) @@ -1220,6 +1353,327 @@ namespace ZEngine::Rendering } } + RenderResourceManager::CookedMeshAdmission RenderResourceManager::RecordCookedMeshUpload(uint8_t frame_index, Meshes::CookedMeshStaging& staging, TLSFSlab& metadata_slab, CookedMeshUploadId* output) + { + if (!output || !m_device || m_scene_resources_sealed.load(std::memory_order_acquire) || staging.m_phase != Meshes::CookedMeshStaging::Phase::Staged || !staging.m_mesh || staging.m_staging.m_device != m_device || !staging.m_staging.Ready() || !m_pool.VertexBuffer || !m_pool.IndexBuffer || m_cooked_mesh_upload_serial == UINT64_MAX) + { + return CookedMeshAdmission::Failed; + } + uint32_t pending_slot = CookedMeshUploadCapacity; + for (uint32_t index = 0; index < CookedMeshUploadCapacity; ++index) + { + if (m_cooked_mesh_uploads[index].State == CookedMeshUploadState::Invalid) + { + pending_slot = index; + break; + } + } + if (pending_slot == CookedMeshUploadCapacity) + { + return CookedMeshAdmission::Retry; + } + const auto batch = TryEnsureCookedBatchOpen(frame_index); + if (batch != CookedMeshAdmission::Recorded) + { + return batch; + } + auto& frame = m_batch_frames[frame_index]; + if (frame.StagingCount >= MAX_PENDING * 2) + { + return CookedMeshAdmission::Retry; + } + const auto* metadata = CaptureMeshDrawMetadata(metadata_slab, *staging.m_mesh); + if (!metadata) + { + return CookedMeshAdmission::Failed; + } + GeometryRegion region = {}; + if (!m_pool.Allocate(staging.m_vertex_bytes, staging.m_index_bytes, region)) + { + ReleaseMeshDrawMetadata(metadata); + return CookedMeshAdmission::Failed; + } + const auto vertices = staging.m_vertex_bytes / DRAW_VERTEX_BYTES; + const auto indices = staging.m_index_bytes / sizeof(uint32_t); + const bool invalid_region = region.VtxByteOffset % DRAW_VERTEX_BYTES || region.IdxByteOffset % sizeof(uint32_t) || region.VtxByteOffset > m_pool.VertexBuffer.Size || region.VtxByteSize > m_pool.VertexBuffer.Size - region.VtxByteOffset || region.IdxByteOffset > m_pool.IndexBuffer.Size || region.IdxByteSize > m_pool.IndexBuffer.Size - region.IdxByteOffset || region.VtxByteOffset / DRAW_VERTEX_BYTES > UINT32_MAX - vertices || region.IdxByteOffset / sizeof(uint32_t) > UINT32_MAX - indices; + const uint64_t version = invalid_region ? 0 : NextMeshGeometryVersion(); + const uint32_t slot = version ? AllocMeshSlot() : UINT32_MAX; + if (slot == UINT32_MAX) + { + m_pool.Free(region); + ReleaseMeshDrawMetadata(metadata); + return CookedMeshAdmission::Failed; + } + MeshSlot mesh = {}; + mesh.Region = region; + mesh.VtxCount = static_cast(vertices); + mesh.IdxCount = static_cast(indices); + mesh.State = StreamingState::Resident; + mesh.Version = version; + mesh.DrawMetadata = metadata; + m_mesh_slots[slot].Data = mesh; + const BufferHandle handle = {slot, m_mesh_slots[slot].Generation}; + auto pin = PinMeshGeometry(handle); + if (!pin) + { + RetireMeshGeometry(m_mesh_slots[slot].Data); + m_mesh_slots[slot] = {}; + return CookedMeshAdmission::Failed; + } + + const auto& source = staging.m_staging.m_buffer; + GlobalBufferCopyCtx vertices_copy{source.Handle, m_pool.VertexBuffer.Handle, region.VtxByteOffset, staging.m_vertex_bytes, 0}; + GlobalBufferCopyCtx indices_copy{source.Handle, m_pool.IndexBuffer.Handle, region.IdxByteOffset, staging.m_index_bytes, staging.m_vertex_bytes}; + RecordGlobalBufferCopy(m_batch_cmd->GetHandle(), &vertices_copy); + RecordGlobalBufferCopy(m_batch_cmd->GetHandle(), &indices_copy); + auto& record = m_cooked_mesh_uploads[pending_slot]; + record.State = CookedMeshUploadState::Pending; + record.Serial = ++m_cooked_mesh_upload_serial; + record.SignalValue = m_batch_next_value + 1; + record.CleanupPending = true; + record.Identity = staging.m_identity; + record.Digest = staging.m_digest; + record.Handle = handle; + record.Pin = std::move(pin); + frame.StagingBuffers[frame.StagingCount++] = std::exchange(staging.m_staging.m_buffer, BufferView{}); + staging.m_staging.m_device = nullptr; + staging.m_staging.m_written = 0; + staging.Reset(); + *output = {pending_slot, record.Serial}; + return CookedMeshAdmission::Recorded; + } + + void RenderResourceManager::ReleaseCookedMeshResources(CookedMeshUploadRecord& record) + { + Release(record.Handle); + record.Handle = {}; + record.Pin.Reset(); + record.CleanupPending = false; + } + + void RenderResourceManager::PollCookedMeshUploads() + { + if (!m_device || !m_batch_timeline) + { + return; + } + bool need_query = false; + for (const auto& record : m_cooked_mesh_uploads) + { + need_query |= record.CleanupPending && record.SignalValue <= m_batch_accepted_value; + } + uint64_t completed = 0; + bool lost = m_device->IsDeviceLost.load(std::memory_order_acquire); + if (need_query && !lost) + { + const VkResult result = vkGetSemaphoreCounterValue(m_device->LogicalDevice, m_batch_timeline->GetHandle(), &completed); + if (result != VK_SUCCESS) + { + lost = m_device->CheckDeviceLost(result, "Cooked mesh upload completion query"); + if (!lost) + { + return; + } + } + } + for (auto& record : m_cooked_mesh_uploads) + { + if (record.State == CookedMeshUploadState::Ready) + { + const auto* snapshot = record.Pin.Get(); + const bool valid = snapshot && record.Handle.IsValid() && record.Handle.Index < m_mesh_slot_count && m_mesh_slots[record.Handle.Index].Generation == record.Handle.Generation && m_mesh_slots[record.Handle.Index].Data.Version == snapshot->Version; + if (!valid) + { + record.State = CookedMeshUploadState::Failed; + ReleaseCookedMeshResources(record); + } + continue; + } + if (!record.CleanupPending) + { + continue; + } + if (record.SignalValue > m_batch_accepted_value) + { + if (m_batch_submission_failed && !m_batch_mode) + { + record.State = record.CancelRequested ? CookedMeshUploadState::Cancelled : CookedMeshUploadState::Failed; + ReleaseCookedMeshResources(record); // The batch was never accepted. + } + continue; + } + if (lost) + { + record.State = record.CancelRequested ? CookedMeshUploadState::Cancelled : CookedMeshUploadState::Failed; + continue; // Accepted work stays owned until joined device shutdown. + } + if (completed < record.SignalValue) + { + continue; + } + const auto* snapshot = record.Pin.Get(); + const bool valid = snapshot && record.Handle.IsValid() && record.Handle.Index < m_mesh_slot_count && m_mesh_slots[record.Handle.Index].Generation == record.Handle.Generation && m_mesh_slots[record.Handle.Index].Data.Version == snapshot->Version; + if (record.CancelRequested || !valid) + { + record.State = record.CancelRequested ? CookedMeshUploadState::Cancelled : CookedMeshUploadState::Failed; + ReleaseCookedMeshResources(record); + } + else + { + record.State = CookedMeshUploadState::Ready; + record.CleanupPending = false; + } + } + } + + RenderResourceManager::CookedMeshUploadResult RenderResourceManager::PollCookedMeshUpload(CookedMeshUploadId id) + { + if (id.Slot >= CookedMeshUploadCapacity || !id.Serial || m_cooked_mesh_uploads[id.Slot].Serial != id.Serial) + { + return {}; + } + PollCookedMeshUploads(); + const auto& record = m_cooked_mesh_uploads[id.Slot]; + return {record.State, record.CleanupPending, record.Identity, record.Digest}; + } + + bool RenderResourceManager::CancelCookedMeshUpload(CookedMeshUploadId id) + { + if (id.Slot >= CookedMeshUploadCapacity || !id.Serial || m_cooked_mesh_uploads[id.Slot].Serial != id.Serial) + { + return false; + } + auto& record = m_cooked_mesh_uploads[id.Slot]; + if (record.State == CookedMeshUploadState::Ready) + { + ReleaseCookedMeshResources(record); + } + record.CancelRequested = true; + record.State = CookedMeshUploadState::Cancelled; + PollCookedMeshUploads(); + return true; + } + + RenderResourceManager::CookedMeshVersion::~CookedMeshVersion() + { + Reset(); + } + + RenderResourceManager::CookedMeshVersion::CookedMeshVersion(CookedMeshVersion&& other) noexcept : m_owner(std::exchange(other.m_owner, nullptr)), m_handle(std::exchange(other.m_handle, {})), m_pin(std::move(other.m_pin)), m_identity(std::exchange(other.m_identity, {})), m_digest(std::exchange(other.m_digest, {})) {} + + RenderResourceManager::CookedMeshVersion& RenderResourceManager::CookedMeshVersion::operator=(CookedMeshVersion&& other) noexcept + { + if (this != &other) + { + Reset(); + m_owner = std::exchange(other.m_owner, nullptr); + m_handle = std::exchange(other.m_handle, {}); + m_pin = std::move(other.m_pin); + m_identity = std::exchange(other.m_identity, {}); + m_digest = std::exchange(other.m_digest, {}); + } + return *this; + } + + void RenderResourceManager::CookedMeshVersion::Reset() + { + if (m_owner) + { + m_owner->Release(m_handle); + m_owner = nullptr; + } + m_pin.Reset(); + m_handle = {}; + m_identity = {}; + m_digest = {}; + } + + bool RenderResourceManager::TakeCookedMeshUpload(CookedMeshUploadId id, CookedMeshVersion* output) + { + if (!output || *output || id.Slot >= CookedMeshUploadCapacity || !id.Serial || m_cooked_mesh_uploads[id.Slot].Serial != id.Serial) + { + return false; + } + PollCookedMeshUploads(); + auto& record = m_cooked_mesh_uploads[id.Slot]; + if (record.State != CookedMeshUploadState::Ready || !record.Pin) + { + return false; + } + output->m_owner = this; + output->m_handle = record.Handle; + output->m_pin = std::move(record.Pin); + output->m_identity = record.Identity; + output->m_digest = record.Digest; + record = {}; + return true; + } + + bool RenderResourceManager::TakeCookedMeshUpload(CookedMeshUploadId id, BufferHandle* handle, MeshGeometryPin* pin) + { + if (!handle || !pin || *pin || id.Slot >= CookedMeshUploadCapacity || !id.Serial || m_cooked_mesh_uploads[id.Slot].Serial != id.Serial) + { + return false; + } + PollCookedMeshUploads(); + auto& record = m_cooked_mesh_uploads[id.Slot]; + if (record.State != CookedMeshUploadState::Ready || !record.Pin) + { + return false; + } + *handle = record.Handle; + *pin = std::move(record.Pin); + record = {}; + return true; + } + + bool RenderResourceManager::ReleaseCookedMeshUpload(CookedMeshUploadId id) + { + if (id.Slot >= CookedMeshUploadCapacity || !id.Serial || m_cooked_mesh_uploads[id.Slot].Serial != id.Serial) + { + return false; + } + PollCookedMeshUploads(); + auto& record = m_cooked_mesh_uploads[id.Slot]; + if (record.State == CookedMeshUploadState::Pending || record.CleanupPending) + { + return false; + } + if (record.Pin) + { + ReleaseCookedMeshResources(record); + } + record = {}; + return true; + } + + bool RenderResourceManager::AbandonCookedMeshUploadAfterDeviceIdle(CookedMeshUploadId id) + { + if (id.Slot >= CookedMeshUploadCapacity || !id.Serial || m_cooked_mesh_uploads[id.Slot].Serial != id.Serial) + { + return false; + } + auto& record = m_cooked_mesh_uploads[id.Slot]; + if (record.Pin) + { + ReleaseCookedMeshResources(record); + } + record = {}; + return true; + } + + void RenderResourceManager::ReleaseCookedMeshUploadsAfterIdle() + { + for (auto& record : m_cooked_mesh_uploads) + { + if (record.Pin) + { + ReleaseCookedMeshResources(record); + } + record = {}; + } + } + RenderResourceManager::MeshSlot RenderResourceManager::AppendMeshData(AssetHandle asset, uint32_t frame_index) { auto* assets = AssetManager::Instance(); @@ -1662,7 +2116,7 @@ namespace ZEngine::Rendering const auto& record = m_texture_versions[handle.Index]; const bool same = record.Snapshot.Handle.Index == handle.Index && record.Snapshot.Handle.Generation == handle.Generation; const auto& producer = m_texture_producer_completions[handle.Index]; - const bool pending = producer.Texture.Index == handle.Index && producer.Texture.Generation == handle.Generation && (producer.PendingSubmission || producer.Timeline || producer.AcquirePending); + const bool pending = producer.Texture.Index == handle.Index && producer.Texture.Generation == handle.Generation && (producer.PendingSubmission || producer.Timeline || producer.AcquirePending || producer.ConsumerTimeline || (producer.ConsumerAcquired && producer.ConsumerFamily != m_device->GraphicFamilyIndex)); return !pending && (!same || (!record.References && !record.DestroyRequested)); } @@ -1679,7 +2133,7 @@ namespace ZEngine::Rendering // Ordinary writes/reconstruction must not replace an image still used // by an accepted upload. Poll outside the admission mutex, then recheck // the exact proof under that mutex before authorizing mutation. - if (!TextureProducerComplete(handle)) + if (!TextureProducerComplete(handle) || !TextureConsumerWorkRetired(handle)) { return false; } @@ -1687,7 +2141,7 @@ namespace ZEngine::Rendering auto& record = m_texture_versions[handle.Index]; const bool same = record.Snapshot.Handle.Index == handle.Index && record.Snapshot.Handle.Generation == handle.Generation; const auto& producer = m_texture_producer_completions[handle.Index]; - const bool pending = producer.Texture.Index == handle.Index && producer.Texture.Generation == handle.Generation && (producer.PendingSubmission || producer.Timeline || producer.AcquirePending); + const bool pending = producer.Texture.Index == handle.Index && producer.Texture.Generation == handle.Generation && (producer.PendingSubmission || producer.Timeline || producer.AcquirePending || producer.ConsumerTimeline || (producer.ConsumerAcquired && producer.ConsumerFamily != m_device->GraphicFamilyIndex)); if (pending || (same && (record.References || record.DestroyRequested))) { return false; @@ -1725,7 +2179,7 @@ namespace ZEngine::Rendering auto& producer = m_texture_producer_completions[handle.Index]; const bool same = record.Snapshot.Handle.Index == handle.Index && record.Snapshot.Handle.Generation == handle.Generation; const bool same_producer = producer.Texture.Index == handle.Index && producer.Texture.Generation == handle.Generation; - if ((same && (record.References || record.DestroyRequested)) || (same_producer && (producer.PendingSubmission || producer.Timeline || producer.AcquirePending)) || !TrackTextureDecode(handle)) + if ((same && (record.References || record.DestroyRequested)) || (same_producer && (producer.PendingSubmission || producer.Timeline || producer.AcquirePending || producer.ConsumerTimeline || (producer.ConsumerAcquired && producer.ConsumerFamily != m_device->GraphicFamilyIndex))) || !TrackTextureDecode(handle)) { return false; } @@ -1739,6 +2193,9 @@ namespace ZEngine::Rendering // of this recording must not erase GPU work that is already in flight. producer.PendingSubmission = true; producer.GraphicsUploadOnly |= graphics_upload; + producer.ConsumerAcquired = false; + producer.ConsumerFamily = VK_QUEUE_FAMILY_IGNORED; + producer.ConsumerDiscarded = false; return true; } @@ -1768,7 +2225,7 @@ namespace ZEngine::Rendering // old generation must not acquire a new lease before the queue drains. record.DestroyRequested = true; const auto& producer = m_texture_producer_completions[handle.Index]; - const bool producer_pending = producer.Texture.Index == handle.Index && producer.Texture.Generation == handle.Generation && (producer.PendingSubmission || producer.Timeline); + const bool producer_pending = producer.Texture.Index == handle.Index && producer.Texture.Generation == handle.Generation && (producer.PendingSubmission || producer.Timeline || producer.ConsumerTimeline); if (!record.References) { const uint64_t bit = uint64_t(1) << (handle.Index % 64); @@ -1988,6 +2445,58 @@ namespace ZEngine::Rendering return pin; } + SceneTextureDescriptorState RenderResourceManager::PrepareExplicitMaterialVersion(const Meshes::MeshMaterial& material, const TextureVersionPin* const (&textures)[5], MaterialVersionPin* output) + { + if (!output || *output || m_material_version_sequence == UINT64_MAX || m_scene_resources_sealed.load(std::memory_order_acquire)) + { + return SceneTextureDescriptorState::Failed; + } + const uint32_t maps[] = {material.AlbedoMap, material.EmissiveMap, material.NormalMap, material.OpacityMap, material.SpecularMap}; + MaterialVersionPin candidate = {}; + for (uint32_t map = 0; map < 5; ++map) + { + if (maps[map] == INVALID_MAP_HANDLE) + { + if (textures[map]) + { + return SceneTextureDescriptorState::Failed; + } + continue; + } + const auto* source = textures[map]; + const auto* snapshot = source ? source->Get() : nullptr; + if (!snapshot || source->m_owner != this || snapshot->Handle.Index != maps[map]) + { + return SceneTextureDescriptorState::Failed; + } + const auto acquire = PollSubmittedTextureAcquire(*source); + if (acquire != SceneTextureDescriptorState::Ready) + { + return acquire; + } + candidate.m_textures[map] = CloneTextureVersion(*source); + if (!candidate.m_textures[map]) + { + return SceneTextureDescriptorState::Failed; + } + } + const auto descriptors = PrepareMaterialDescriptorVersions(candidate); + if (descriptors != SceneTextureDescriptorState::Ready) + { + return descriptors; + } + for (uint32_t map = 0; map < 5; ++map) + { + if (candidate.m_textures[map] && !candidate.m_descriptor_versions[map]) + { + return SceneTextureDescriptorState::Failed; + } + } + candidate.m_snapshot = {material, ++m_material_version_sequence}; + *output = std::move(candidate); + return SceneTextureDescriptorState::Ready; + } + bool RenderResourceManager::GetFrameMaterialVersion(const uuids::uuid& uuid, uint32_t* index, MaterialVersionSnapshot* snapshot, bool use_fallback) { auto* assets = AssetManager::Instance(); @@ -2153,6 +2662,92 @@ namespace ZEngine::Rendering return CloneTextureVersion(retained); } + SceneTextureDescriptorState RenderResourceManager::PrepareSubmittedTextureVersion(Textures::TextureHandle handle, TextureVersionPin* retained, uint64_t* publication) + { + if (!retained || !publication || !m_device || !m_device->LogicalDevice || !handle.Valid() || handle.Index >= TextureVersionCapacity || m_device->IsDeviceLost.load(std::memory_order_acquire) || m_scene_resources_sealed.load(std::memory_order_acquire) || !m_scene_descriptor_preparation || TextureReleaseRequested(handle)) + { + return SceneTextureDescriptorState::Failed; + } + const auto* texture = GetTexture(handle); + const auto* image = texture ? m_device->ImageBufferManager.Access(texture->BufferHandle) : nullptr; + if (!texture || !image || texture->Specification.IsAliasable || !image->GetBuffer().OwnsAllocation) + { + return SceneTextureDescriptorState::Failed; + } + const auto decode = GetTextureDecodeState(handle); + if (decode == TextureDecodeState::Failed || decode == TextureDecodeState::Untracked) + { + return SceneTextureDescriptorState::Failed; + } + if (decode == TextureDecodeState::Pending) + { + return SceneTextureDescriptorState::Pending; + } + if (retained->m_owner && (retained->m_owner != this || !retained->Get() || retained->Get()->Handle != handle)) + { + return SceneTextureDescriptorState::Failed; + } + if (!*retained) + { + auto pin = PinTextureVersion(handle); + if (!pin) + { + std::lock_guard lock(m_texture_version_mutex); + const auto& record = m_texture_versions[handle.Index]; + const bool same = record.Snapshot.Handle == handle; + return m_texture_version_sequence == UINT64_MAX || (same && (record.DestroyRequested || record.References == UINT32_MAX)) ? SceneTextureDescriptorState::Failed : SceneTextureDescriptorState::Pending; + } + *retained = std::move(pin); + } + bool failed = false; + if (!TextureProducerComplete(handle, &failed)) + { + return failed ? SceneTextureDescriptorState::Failed : SceneTextureDescriptorState::Pending; + } + const auto* snapshot = retained->Get(); + if (!snapshot || snapshot->Image != texture->BufferHandle) + { + return SceneTextureDescriptorState::Failed; + } + const auto state = m_scene_descriptor_preparation->PrepareSceneTextureDescriptor(handle, snapshot->Image, publication); + if (state != SceneTextureDescriptorState::Ready) + { + return state; + } + return IsSubmittedTextureDescriptorCurrent(*retained, *publication) ? SceneTextureDescriptorState::Ready : SceneTextureDescriptorState::Failed; + } + + bool RenderResourceManager::IsSubmittedTextureDescriptorCurrent(const TextureVersionPin& retained, uint64_t publication) + { + const auto* snapshot = retained.Get(); + return retained.m_owner == this && snapshot && publication && m_device && !m_device->IsDeviceLost.load(std::memory_order_acquire) && !m_scene_resources_sealed.load(std::memory_order_acquire) && !TextureReleaseRequested(snapshot->Handle) && m_scene_descriptor_preparation && m_scene_descriptor_preparation->IsSceneTextureDescriptorCurrent(snapshot->Handle, snapshot->Image, publication); + } + + SceneTextureDescriptorState RenderResourceManager::PollSubmittedTextureAcquire(const TextureVersionPin& retained) + { + const auto* snapshot = retained.Get(); + if (retained.m_owner != this || !snapshot || !m_device || m_device->IsDeviceLost.load(std::memory_order_acquire) || TextureReleaseRequested(snapshot->Handle)) + { + return SceneTextureDescriptorState::Failed; + } + bool failed = false; + if (!TextureConsumerWorkRetired(snapshot->Handle, &failed)) + { + return failed ? SceneTextureDescriptorState::Failed : SceneTextureDescriptorState::Pending; + } + std::lock_guard lock(m_texture_version_mutex); + const auto& record = m_texture_producer_completions[snapshot->Handle.Index]; + if (record.Texture != snapshot->Handle || record.ConsumerDiscarded) + { + return SceneTextureDescriptorState::Failed; + } + if (!record.ConsumerAcquired) + { + return SceneTextureDescriptorState::Pending; + } + return record.ConsumerFamily == m_device->GraphicFamilyIndex ? SceneTextureDescriptorState::Ready : SceneTextureDescriptorState::Failed; + } + bool RenderResourceManager::GetFrameTextureVersion(const TextureVersionPin& retained, TextureVersionSnapshot* snapshot) { const auto* source = retained.Get(); @@ -2554,6 +3149,48 @@ namespace ZEngine::Rendering return true; } PollFrameResources(); + { + // Acquire receipts may reference a graph timeline even when no + // frame resource was stamped on it. Do not let its owner destroy + // the semaphore until every such consumer has actually retired. + std::lock_guard lock(m_texture_version_mutex); + bool referenced = false; + for (const auto& record : m_texture_producer_completions) + { + referenced |= record.ConsumerTimeline == timeline; + } + if (referenced) + { + if (!m_device || !m_device->LogicalDevice) + { + return false; + } + uint64_t completed = 0; + const VkResult result = vkGetSemaphoreCounterValue(m_device->LogicalDevice, timeline->GetHandle(), &completed); + const bool lost = m_device->CheckDeviceLost(result, "Texture consumer timeline retirement") || m_device->IsDeviceLost.load(std::memory_order_acquire); + if (result != VK_SUCCESS && !lost) + { + return false; + } + for (const auto& record : m_texture_producer_completions) + { + if (record.ConsumerTimeline == timeline && !lost && completed < record.ConsumerValue) + { + return false; + } + } + for (auto& record : m_texture_producer_completions) + { + if (record.ConsumerTimeline == timeline) + { + record.ConsumerTimeline = nullptr; + record.ConsumerValue = 0; + record.ConsumerAcquired = !lost; + record.ConsumerDiscarded = lost; + } + } + } + } for (uint32_t i = 0; i < ResourceTimelineCount; ++i) { if (m_resource_timelines[i] != timeline) @@ -2774,7 +3411,24 @@ namespace ZEngine::Rendering return false; } const auto state = record.Environment.load(std::memory_order_acquire); - *out = {id, record.EnvironmentInput, record.ExpectedGeneration, state, phase == SceneResourcePhase::Failed || (phase == SceneResourcePhase::Releasing && state != SceneEnvironmentPreparationState::Activated && record.Readers.load(std::memory_order_acquire) <= 1)}; + *out = {id, record.EnvironmentInput, record.ExpectedGeneration, state, phase == SceneResourcePhase::Failed || (phase == SceneResourcePhase::Releasing && state != SceneEnvironmentPreparationState::Activated && record.Readers.load(std::memory_order_acquire) <= 1), record.CookedEnvironment, record.CookedEnvironmentFace, record.CookedEnvironmentHash}; + return true; + } + + bool RenderResourceManager::TakeCookedSceneEnvironment(SceneResourcePreparationId id, Textures::TextureHandle* out) + { + SceneEnvironmentPreparation preparation = {}; + if (!out || out->Valid() || !GetSceneEnvironmentPreparation(id, &preparation) || preparation.Cancelled || preparation.State != SceneEnvironmentPreparationState::Preparing) + { + return false; + } + auto& record = m_scene_resources[id.Slot]; + if (!record.CookedEnvironmentOwned || !record.CookedEnvironmentPin) + { + return false; + } + *out = record.CookedEnvironment; + record.CookedEnvironmentOwned = false; return true; } @@ -2814,9 +3468,10 @@ namespace ZEngine::Rendering } } - bool RenderResourceManager::PrepareSceneResources(ECS::SceneRequestId request, uint64_t expected_generation, std::span meshes, SceneResourcePreparationId* out, ScenePreparationContent content, const SceneEnvironmentInput* environment, const SceneInitialFrameInput* initial_frame, bool defer_instances) + bool RenderResourceManager::PrepareSceneResources(ECS::SceneRequestId request, uint64_t expected_generation, std::span meshes, SceneResourcePreparationId* out, ScenePreparationContent content, const SceneEnvironmentInput* environment, const SceneInitialFrameInput* initial_frame, bool defer_instances, const CookedCpuAssetSet* cooked) { - if (!request || !expected_generation || expected_generation == UINT64_MAX || !out || *out || meshes.size() > SceneGeometryMeshCapacity || m_scene_resources_sequence == UINT64_MAX || m_scene_resources_sealed.load(std::memory_order_acquire) || (content != ScenePreparationContent::GeometryOnly && content != ScenePreparationContent::MeshMaterials) || (environment && environment->SkyRevision == 0) || (initial_frame && (content != ScenePreparationContent::MeshMaterials || initial_frame->Instances.size() > SceneDrawCapacity))) + if (!request || !expected_generation || expected_generation == UINT64_MAX || !out || *out || meshes.size() > SceneGeometryMeshCapacity || m_scene_resources_sequence == UINT64_MAX || m_scene_resources_sealed.load(std::memory_order_acquire) || (content != ScenePreparationContent::GeometryOnly && content != ScenePreparationContent::MeshMaterials) || (environment && environment->SkyRevision == 0) || (initial_frame && (content != ScenePreparationContent::MeshMaterials || initial_frame->Instances.size() > SceneDrawCapacity)) || + (cooked && (!cooked->IsValid() || content != ScenePreparationContent::MeshMaterials))) { return false; } @@ -2874,10 +3529,19 @@ namespace ZEngine::Rendering record.ExpectedGeneration = expected_generation; record.MeshCount = static_cast(meshes.size()); std::copy(meshes.begin(), meshes.end(), record.Meshes); - record.Cursor = 0; - record.ResidencyWait = {}; - record.Content = content; - record.EnvironmentInput = environment ? *environment : SceneEnvironmentInput{}; + record.Cursor = 0; + record.Cooked = cooked != nullptr; + record.CookedManifest = cooked ? cooked->ManifestDigest() : Core::Hash::Digest256{}; + record.CookedScene = cooked ? cooked->SceneUUID() : uuids::uuid{}; + record.CookedMaterialCount = 0; + record.CookedTextureCount = 0; + record.CookedEnvironment = {}; + record.CookedEnvironmentFace = 0; + record.CookedEnvironmentHash = 0; + record.CookedEnvironmentOwned = false; + record.ResidencyWait = {}; + record.Content = content; + record.EnvironmentInput = environment ? *environment : SceneEnvironmentInput{}; record.Environment.store(environment ? SceneEnvironmentPreparationState::Queued : SceneEnvironmentPreparationState::NotRequested, std::memory_order_relaxed); record.EnvironmentError.store(SceneResourcePreparationError::None, std::memory_order_relaxed); record.MaterialCount = 0; @@ -2905,57 +3569,176 @@ namespace ZEngine::Rendering record.Error = SceneResourcePreparationError::None; record.MaterialError = MaterialVersionError::None; *out = record.Id; - record.Phase.store(record.InitialCopyCount < record.InitialInstanceCount ? SceneResourcePhase::Collecting : SceneResourcePhase::Queued, std::memory_order_release); + record.Phase.store(record.Cooked || record.InitialCopyCount < record.InitialInstanceCount ? SceneResourcePhase::Collecting : SceneResourcePhase::Queued, std::memory_order_release); return true; } - return false; + return false; + } + + bool RenderResourceManager::AppendSceneResourceInstances(SceneResourcePreparationId id, std::span instances) + { + if (!id || id.Slot >= SceneResourcePreparationCapacity) + { + return false; + } + auto& record = m_scene_resources[id.Slot]; + if (record.Phase.load(std::memory_order_acquire) != SceneResourcePhase::Collecting || record.Id != id) + { + return false; + } + bool valid = !instances.empty() && instances.size() <= 64 && instances.size() <= record.InitialInstanceCount - record.InitialCopyCount; + if (!valid) + { + record.Error = SceneResourcePreparationError::InvalidInitialFrame; + record.Phase.store(SceneResourcePhase::Failed, std::memory_order_release); + return false; + } + for (const auto& instance : instances) + { + valid &= !instance.MeshUUID.is_nil() && std::binary_search(record.Meshes, record.Meshes + record.MeshCount, instance.MeshUUID); + for (uint32_t row = 0; row < 4; ++row) + { + for (uint32_t column = 0; column < 4; ++column) + { + valid &= std::isfinite(instance.Transform(row, column)); + } + } + } + if (!valid) + { + record.Error = SceneResourcePreparationError::InvalidInitialFrame; + record.Phase.store(SceneResourcePhase::Failed, std::memory_order_release); + return false; + } + for (const auto& instance : instances) + { + const auto index = record.InitialCopyCount++; + record.InitialTransforms[index] = instance.Transform; + record.InitialInstances[index] = {instance.Entity, static_cast(std::lower_bound(record.Meshes, record.Meshes + record.MeshCount, instance.MeshUUID) - record.Meshes)}; + } + if (record.InitialCopyCount == record.InitialInstanceCount && !record.Cooked) + { + record.Phase.store(SceneResourcePhase::Queued, std::memory_order_release); + } + return true; + } + + bool RenderResourceManager::AppendCookedSceneMesh(SceneResourcePreparationId id, const CookedCpuAssetSet& assets, const CookedMeshVersion& mesh) + { + if (!id || id.Slot >= SceneResourcePreparationCapacity) + { + return false; + } + auto& record = m_scene_resources[id.Slot]; + const auto* version = assets.FindVersion(mesh.Identity()); + if (record.Phase.load(std::memory_order_acquire) != SceneResourcePhase::Collecting || record.Id != id || !record.Cooked || !assets.IsValid() || assets.ManifestDigest() != record.CookedManifest || assets.SceneUUID() != record.CookedScene || !version || !assets.FindMesh(mesh.Identity()) || version->ObjectDigest != mesh.Digest() || mesh.m_owner != this || record.Cursor >= record.MeshCount || record.Meshes[record.Cursor] != mesh.Identity().UUID) + { + return false; + } + auto pin = PinMeshGeometry(mesh.m_handle); + if (!pin || !mesh.Geometry() || pin.Get()->Version != mesh.Geometry()->Version) + { + return false; + } + record.Pins[record.Cursor++] = std::move(pin); + return true; + } + + bool RenderResourceManager::AppendCookedSceneMaterial(SceneResourcePreparationId id, const CookedCpuAssetSet& assets, const Materials::CookedMaterialVersion& material) + { + if (!id || id.Slot >= SceneResourcePreparationCapacity) + { + return false; + } + auto& record = m_scene_resources[id.Slot]; + const auto* version = assets.FindVersion(material.Identity()); + if (record.Phase.load(std::memory_order_acquire) != SceneResourcePhase::Collecting || record.Id != id || !record.Cooked || !assets.IsValid() || assets.ManifestDigest() != record.CookedManifest || assets.SceneUUID() != record.CookedScene || !version || !assets.FindMaterial(material.Identity()) || version->ObjectDigest != material.Digest() || !material || record.CookedMaterialCount >= SceneMaterialCapacity || (record.CookedMaterialCount && !(record.CookedMaterials[record.CookedMaterialCount - 1] < material.Identity().UUID))) + { + return false; + } + for (const auto& texture : material.Pin().m_textures) + { + if (texture.m_owner && texture.m_owner != this) + { + return false; + } + } + auto pin = CloneMaterialVersion(material.Pin()); + if (!pin || !MaterialDescriptorVersionsCurrent(pin)) + { + return false; + } + record.CookedMaterials[record.CookedMaterialCount] = material.Identity().UUID; + record.CookedMaterialPins[record.CookedMaterialCount++] = std::move(pin); + return true; } - bool RenderResourceManager::AppendSceneResourceInstances(SceneResourcePreparationId id, std::span instances) + bool RenderResourceManager::AppendCookedSceneTexture(SceneResourcePreparationId id, const CookedCpuAssetSet& assets, Textures::CookedTextureUpload& upload) { if (!id || id.Slot >= SceneResourcePreparationCapacity) { return false; } - auto& record = m_scene_resources[id.Slot]; - if (record.Phase.load(std::memory_order_acquire) != SceneResourcePhase::Collecting || record.Id != id) + auto& record = m_scene_resources[id.Slot]; + const auto* version = assets.FindVersion(upload.Identity()); + const auto* binding = upload.ReadyBinding(); + if (record.Phase.load(std::memory_order_acquire) != SceneResourcePhase::Collecting || record.Id != id || !record.Cooked || !assets.IsValid() || assets.ManifestDigest() != record.CookedManifest || assets.SceneUUID() != record.CookedScene || !version || !assets.FindTexture(upload.Identity()) || version->ObjectDigest != upload.Digest() || !upload.BelongsTo(*this) || !binding || !binding->Handle.Valid() || record.CookedTextureCount == SceneCookedTextureCapacity || + (record.CookedTextureCount && !(record.CookedTextures[record.CookedTextureCount - 1] < upload.Identity().UUID))) { return false; } - bool valid = !instances.empty() && instances.size() <= 64 && instances.size() <= record.InitialInstanceCount - record.InitialCopyCount; - if (!valid) + auto pin = upload.RetainPreparedVersion(); + if (!pin) { - record.Error = SceneResourcePreparationError::InvalidInitialFrame; - record.Phase.store(SceneResourcePhase::Failed, std::memory_order_release); return false; } - for (const auto& instance : instances) + const auto index = record.CookedTextureCount++; + record.CookedTextures[index] = upload.Identity().UUID; + record.CookedTextureHandles[index] = binding->Handle; + record.CookedTexturePins[index] = std::move(pin); + return true; + } + + bool RenderResourceManager::FinishCookedSceneResources(SceneResourcePreparationId id) + { + if (!id || id.Slot >= SceneResourcePreparationCapacity) { - valid &= !instance.MeshUUID.is_nil() && std::binary_search(record.Meshes, record.Meshes + record.MeshCount, instance.MeshUUID); - for (uint32_t row = 0; row < 4; ++row) - { - for (uint32_t column = 0; column < 4; ++column) - { - valid &= std::isfinite(instance.Transform(row, column)); - } - } + return false; } - if (!valid) + auto& record = m_scene_resources[id.Slot]; + if (record.Phase.load(std::memory_order_acquire) != SceneResourcePhase::Collecting || record.Id != id || !record.Cooked || record.Cursor != record.MeshCount || record.InitialCopyCount != record.InitialInstanceCount || (record.EnvironmentInput.Sky.IsHDRI() && !record.CookedEnvironmentOwned)) { - record.Error = SceneResourcePreparationError::InvalidInitialFrame; - record.Phase.store(SceneResourcePhase::Failed, std::memory_order_release); return false; } - for (const auto& instance : instances) + auto expected = SceneResourcePhase::Collecting; + return record.Phase.compare_exchange_strong(expected, SceneResourcePhase::Queued, std::memory_order_release); + } + + bool RenderResourceManager::AppendCookedSceneEnvironment(SceneResourcePreparationId id, const CookedCpuAssetSet& assets, const Textures::CookedEnvironmentUpload& upload) + { + if (!id || id.Slot >= SceneResourcePreparationCapacity) { - const auto index = record.InitialCopyCount++; - record.InitialTransforms[index] = instance.Transform; - record.InitialInstances[index] = {instance.Entity, static_cast(std::lower_bound(record.Meshes, record.Meshes + record.MeshCount, instance.MeshUUID) - record.Meshes)}; + return false; } - if (record.InitialCopyCount == record.InitialInstanceCount) + auto& record = m_scene_resources[id.Slot]; + const auto* version = assets.FindVersion(upload.Identity()); + const auto* header = assets.FindEnvironment(upload.Identity()); + const auto texture = upload.SubmittedTexture(); + if (record.Phase.load(std::memory_order_acquire) != SceneResourcePhase::Collecting || record.Id != id || !record.Cooked || !record.EnvironmentInput.Sky.IsHDRI() || record.EnvironmentInput.Sky.EnvironmentMap != upload.Identity().UUID || record.CookedEnvironmentOwned || !assets.IsValid() || assets.ManifestDigest() != record.CookedManifest || assets.SceneUUID() != record.CookedScene || !version || !header || !upload.Header() || version->ObjectDigest != upload.Digest() || !upload.BelongsTo(*this) || !texture.Valid() || + GetTextureDecodeState(texture) != TextureDecodeState::Succeeded) { - record.Phase.store(SceneResourcePhase::Queued, std::memory_order_release); + return false; + } + auto pin = PinTextureVersion(texture); + if (!pin) + { + return false; } + record.CookedEnvironment = texture; + record.CookedEnvironmentFace = header->FaceWidth; + record.CookedEnvironmentHash = header->SourceHash; + record.CookedEnvironmentPin = std::move(pin); + record.CookedEnvironmentOwned = true; return true; } @@ -2973,6 +3756,9 @@ namespace ZEngine::Rendering } SceneResourcePreparationResult result = {id, record.ExpectedGeneration, record.MeshCount}; result.Content = record.Content; + result.Cooked = record.Cooked; + result.CookedManifest = record.CookedManifest; + result.CookedScene = record.CookedScene; result.FrameRecorded = record.FrameRecorded.load(std::memory_order_acquire); result.Environment = record.Environment.load(std::memory_order_acquire); if (phase == SceneResourcePhase::Ready) @@ -3212,7 +3998,7 @@ namespace ZEngine::Rendering { return; } - if (!failed && m_batch_timeline) + if (!m_device->IsDeviceLost.load(std::memory_order_acquire) && m_batch_timeline) { const VkResult result = vkGetSemaphoreCounterValue(m_device->LogicalDevice, m_batch_timeline->GetHandle(), &completed); if (result != VK_SUCCESS) @@ -3224,10 +4010,30 @@ namespace ZEngine::Rendering } } } - TickSceneResourcePreparations(completed, failed || m_device->IsDeviceLost.load(std::memory_order_acquire)); + const bool device_lost = m_device->IsDeviceLost.load(std::memory_order_acquire); + TickSceneResourcePreparations(completed, failed || device_lost, device_lost); + if (m_batch_submission_failed && !device_lost && !m_batch_mode) + { + // Keep the rejection sticky until every affected owner has seen a + // terminal result and released its payload. Otherwise a later + // accepted signal could make an unsubmitted value look complete. + PollCookedMeshUploads(); + bool outstanding = false; + for (const auto& upload : m_cooked_mesh_uploads) + { + outstanding |= upload.State != CookedMeshUploadState::Invalid; + } + for (const auto& record : m_scene_resources) + { + const auto phase = record.Phase.load(std::memory_order_acquire); + outstanding |= phase != SceneResourcePhase::Free && phase != SceneResourcePhase::Ready; + } + if (!outstanding) + m_batch_submission_failed = false; + } } - void RenderResourceManager::TickSceneResourcePreparations(uint64_t completed, bool upload_failed) + void RenderResourceManager::TickSceneResourcePreparations(uint64_t completed, bool upload_failed, bool invalidate_ready) { ZENGINE_PROFILE_SCOPE("SceneLoad.Render.ResourcePreparation"); uint32_t budget = SceneResourceSliceOperations; @@ -3236,7 +4042,7 @@ namespace ZEngine::Rendering { auto& record = m_scene_resources[(first + offset) % SceneResourcePreparationCapacity]; auto phase = record.Phase.load(std::memory_order_acquire); - if (upload_failed && phase == SceneResourcePhase::Ready) + if (upload_failed && invalidate_ready && phase == SceneResourcePhase::Ready) { record.Error = SceneResourcePreparationError::UploadFailed; record.Phase.compare_exchange_strong(phase, SceneResourcePhase::Failed, std::memory_order_release); @@ -3258,6 +4064,12 @@ namespace ZEngine::Rendering { continue; } + // A rejected batch has no completion value, but earlier + // accepted work still needs its own retirement gate. + if (!invalidate_ready && upload_failed && record.UploadValue <= m_batch_accepted_value && !upload_complete()) + { + continue; + } if (!upload_failed && !upload_complete()) { continue; @@ -3286,12 +4098,34 @@ namespace ZEngine::Rendering record.MaterialPins[index].Reset(); --budget; } - while (!record.MaterialCount && record.Cursor && budget) + if (!record.MaterialCount && record.CookedEnvironmentPin && budget) + { + if (record.CookedEnvironmentOwned) + { + ReleaseTexture(record.CookedEnvironment); + record.CookedEnvironmentOwned = false; + } + record.CookedEnvironmentPin.Reset(); + --budget; + } + while (!record.MaterialCount && !record.CookedEnvironmentPin && record.CookedMaterialCount && budget) + { + record.CookedMaterialPins[--record.CookedMaterialCount].Reset(); + --budget; + } + while (!record.MaterialCount && !record.CookedEnvironmentPin && !record.CookedMaterialCount && record.CookedTextureCount && budget) + { + const auto index = --record.CookedTextureCount; + ReleaseTexture(record.CookedTextureHandles[index]); + record.CookedTexturePins[index].Reset(); + --budget; + } + while (!record.MaterialCount && !record.CookedMaterialCount && !record.CookedTextureCount && record.Cursor && budget) { record.Pins[--record.Cursor].Reset(); --budget; } - if (!record.Cursor && !record.MaterialCount) + if (!record.Cursor && !record.MaterialCount && !record.CookedMaterialCount && !record.CookedEnvironmentPin && !record.CookedTextureCount) { // Identity/input stay untouched until the producer claims Free. record.Phase.store(SceneResourcePhase::Free, std::memory_order_release); @@ -3419,7 +4253,21 @@ namespace ZEngine::Rendering break; } MaterialVersionError material_error = MaterialVersionError::None; - auto pin = PinMaterialVersion(material, MaterialVersionPolicy::RequiredSceneBindings, &material_error); + MaterialVersionPin pin = {}; + if (record.Cooked) + { + const auto* found = std::lower_bound(record.CookedMaterials, record.CookedMaterials + record.CookedMaterialCount, material); + if (found == record.CookedMaterials + record.CookedMaterialCount || *found != material) + { + record.Error = SceneResourcePreparationError::RequiredMaterialBinding; + break; + } + pin = CloneMaterialVersion(record.CookedMaterialPins[found - record.CookedMaterials]); + } + else + { + pin = PinMaterialVersion(material, MaterialVersionPolicy::RequiredSceneBindings, &material_error); + } if (!pin) { if (material_error == MaterialVersionError::ResourcesUnavailable && HasPendingTextureUploads()) @@ -3660,6 +4508,42 @@ namespace ZEngine::Rendering return true; } + bool RenderResourceManager::TextureConsumerWorkRetired(Textures::TextureHandle handle, bool* failed) + { + if (!handle.Valid() || handle.Index >= TextureVersionCapacity) + { + return true; + } + // Keep the timeline borrowed while ForgetResourceTimeline checks and + // clears these records before its owner can destroy the semaphore. + std::lock_guard lock(m_texture_version_mutex); + auto& record = m_texture_producer_completions[handle.Index]; + auto* timeline = record.Texture == handle ? record.ConsumerTimeline : nullptr; + if (!timeline) + { + return true; + } + uint64_t completed = 0; + const VkResult result = vkGetSemaphoreCounterValue(m_device->LogicalDevice, timeline->GetHandle(), &completed); + if (result != VK_SUCCESS) + { + m_device->CheckDeviceLost(result, "Texture consumer completion query"); + if (failed) + { + *failed = true; + } + return false; + } + if (completed < record.ConsumerValue) + { + return false; + } + record.ConsumerTimeline = nullptr; + record.ConsumerValue = 0; + record.ConsumerAcquired = true; + return true; + } + bool RenderResourceManager::HasPreparedSceneBuffers(bool pending_only) const { for (const auto& record : m_scene_resources) @@ -4074,6 +4958,22 @@ namespace ZEngine::Rendering ReleaseFrameMaterialReservation(record.FrameMaterialIndices[index]); record.MaterialPins[index].Reset(); } + while (record.CookedMaterialCount) + { + record.CookedMaterialPins[--record.CookedMaterialCount].Reset(); + } + if (record.CookedEnvironmentOwned) + { + ReleaseTexture(record.CookedEnvironment); + record.CookedEnvironmentOwned = false; + } + record.CookedEnvironmentPin.Reset(); + while (record.CookedTextureCount) + { + const auto index = --record.CookedTextureCount; + ReleaseTexture(record.CookedTextureHandles[index]); + record.CookedTexturePins[index].Reset(); + } while (record.Cursor) { record.Pins[--record.Cursor].Reset(); @@ -4537,12 +5437,143 @@ namespace ZEngine::Rendering } } + RenderResourceManager::UploadStaging::~UploadStaging() + { + Reset(); + } + + RenderResourceManager::UploadStaging::UploadStaging(UploadStaging&& other) noexcept : m_device(std::exchange(other.m_device, nullptr)), m_buffer(std::exchange(other.m_buffer, BufferView{})), m_written(std::exchange(other.m_written, 0)) {} + + RenderResourceManager::UploadStaging& RenderResourceManager::UploadStaging::operator=(UploadStaging&& other) noexcept + { + if (this != &other) + { + Reset(); + m_device = std::exchange(other.m_device, nullptr); + m_buffer = std::exchange(other.m_buffer, BufferView{}); + m_written = std::exchange(other.m_written, 0); + } + return *this; + } + + void RenderResourceManager::UploadStaging::Reset() noexcept + { + if (m_device && m_buffer) + { + m_device->GpuMem.FreeBuffer(m_buffer); + } + m_device = nullptr; + m_buffer = {}; + m_written = 0; + } + + bool RenderResourceManager::UploadStaging::Append(std::span bytes, size_t* consumed, VkResult* status) + { + if (status) + { + *status = VK_ERROR_INITIALIZATION_FAILED; + } + if (consumed) + { + *consumed = 0; + } + if (!consumed || !m_device || !m_buffer || bytes.empty() || m_written >= m_buffer.Size) + { + return false; + } + const size_t count = std::min({bytes.size(), MaxSliceBytes, static_cast(m_buffer.Size - m_written)}); + VkResult result = VK_SUCCESS; + if (!m_device->GpuMem.WriteHostBuffer(m_buffer, bytes.data(), count, m_written, &result)) + { + m_device->CheckDeviceLost(result, "Texture staging write"); + if (status) + { + *status = result; + } + return false; + } + m_written += count; + *consumed = count; + if (status) + { + *status = VK_SUCCESS; + } + return true; + } + + bool RenderResourceManager::PrepareUploadStaging(VkDeviceSize size, UploadStaging* output, VkResult* status) + { + if (status) + { + *status = VK_ERROR_INITIALIZATION_FAILED; + } + if (!m_device || !output || !size || size > SIZE_MAX) + { + return false; + } + if (m_device->IsDeviceLost.load(std::memory_order_acquire)) + { + if (status) + { + *status = VK_ERROR_DEVICE_LOST; + } + return false; + } + VkResult result = VK_SUCCESS; + BufferView buffer = m_device->GpuMem.TryAllocateBuffer(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, GpuMemoryDomain::HostStaging, "RRM::UploadStaging", &result); + if (!buffer) + { + m_device->CheckDeviceLost(result, "Texture staging allocation"); + if (status) + { + *status = result; + } + return false; + } + UploadStaging candidate = {}; + candidate.m_device = m_device; + candidate.m_buffer = buffer; + *output = std::move(candidate); + if (status) + { + *status = VK_SUCCESS; + } + return true; + } + Rendering::Textures::TextureHandle RenderResourceManager::UploadTextureBuffer(uint8_t frame_index, uint8_t thread_index, const Rendering::Textures::TextureHandle& handle, unsigned char* data, size_t data_size) { - using namespace Rendering::Specifications; - using namespace Rendering::Primitives; + if (!m_device || !handle.Valid() || !data || !TextureProducerComplete(handle) || !TextureConsumerWorkRetired(handle) || !CanWriteTextureVersion(handle)) + { + return {}; + } + const auto* texture = m_device->GlobalTextures.Access(handle); + const VkDeviceSize available = texture && !data_size ? texture->BufferSize : static_cast(data_size); + VkDeviceSize size = 0; + if (!texture || !texture->ContainsBaseMip(available, &size) || size > SIZE_MAX) + { + return {}; + } + UploadStaging staging = {}; + if (!PrepareUploadStaging(size, &staging)) + { + return {}; + } + while (!staging.Ready()) + { + const size_t offset = static_cast(staging.Written()); + size_t consumed = 0; + if (!staging.Append({data + offset, static_cast(size) - offset}, &consumed)) + { + return {}; + } + } + return UploadPreparedTextureBuffer(frame_index, thread_index, handle, staging); + } - if (!handle.Valid() || !data || !TextureProducerComplete(handle) || !CanWriteTextureVersion(handle)) + Rendering::Textures::TextureHandle RenderResourceManager::UploadPreparedTextureBuffer(uint8_t frame_index, uint8_t thread_index, const Rendering::Textures::TextureHandle& handle, UploadStaging& prepared) + { + if (!m_device || prepared.m_device != m_device || !prepared.Ready() || !handle.Valid() || !TextureProducerComplete(handle) || !TextureConsumerWorkRetired(handle) || !CanWriteTextureVersion(handle)) { return {}; } @@ -4553,15 +5584,23 @@ namespace ZEngine::Rendering return {}; } + if (!m_device->CommandBufferMgr || thread_index >= m_device->CommandBufferMgr->TotalThreadCount || !m_async_uploads.HasCapacity()) + { + return {}; + } uint32_t pool_index = (frame_index * m_device->CommandBufferMgr->TotalThreadCount) + thread_index; + if (pool_index >= m_device->CommandBufferMgr->TotalPoolCount) + { + return {}; + } - auto texture = m_device->GlobalTextures.Access(handle); + auto texture = m_device->GlobalTextures.Access(handle); if (!texture) { return {}; } - const VkDeviceSize upload_size = data_size == 0 ? texture->BufferSize : static_cast(data_size); - if (upload_size == 0 || upload_size > texture->BufferSize) + VkDeviceSize upload_size = 0; + if (!texture->ContainsBaseMip(prepared.Size(), &upload_size) || prepared.Size() != upload_size) { return {}; } @@ -4570,30 +5609,35 @@ namespace ZEngine::Rendering { return {}; } - auto img_buf_aspect = (texture->Specification.Format == ImageFormat::DEPTH_STENCIL_FROM_DEVICE) ? VK_IMAGE_ASPECT_DEPTH_BIT : VK_IMAGE_ASPECT_COLOR_BIT; - auto buffer_handle = img_buf->GetHandle(); - - if (TextureUploadQueue(handle, img_buf->Layout) == QueueType::TRANSFER_QUEUE) + auto img_buf_aspect = (texture->Specification.Format == ImageFormat::DEPTH_STENCIL_FROM_DEVICE) ? VK_IMAGE_ASPECT_DEPTH_BIT : VK_IMAGE_ASPECT_COLOR_BIT; + auto buffer_handle = img_buf->GetHandle(); + const bool transfer = TextureUploadQueue(handle, img_buf->Layout) == QueueType::TRANSFER_QUEUE; + auto& retire_values = transfer ? m_tex_transfer_retire[pool_index] : m_tex_retire_values[pool_index]; + uint32_t i = 0; + for (; i < GEOMETRY_UPLOAD_SLOT; ++i) { - auto& transfer_retire = m_tex_transfer_retire[pool_index]; - uint32_t i = 0; - for (; i < GEOMETRY_UPLOAD_SLOT; ++i) + if (retire_values[i] == 0) { - if (transfer_retire[i] == 0) - { - break; - } - } - if (i >= GEOMETRY_UPLOAD_SLOT) - { - ZENGINE_CORE_WARN("[RRM] UploadTextureBuffer: no free transfer slot — upload deferred") - return {}; - } - if (!BeginTextureUploadRecording(handle, false)) - { - return {}; + break; } + } + if (i >= GEOMETRY_UPLOAD_SLOT) + { + return {}; + } + + if (!BeginTextureUploadRecording(handle, !transfer)) + { + return {}; + } + // The producer slot owns this allocation from here, including inline + // queue rejection and later GPU completion. + BufferView staging = std::exchange(prepared.m_buffer, BufferView{}); + prepared.m_device = nullptr; + prepared.m_written = 0; + if (transfer) + { auto transfer_cmd = m_device->CommandBufferMgr->GetInstantCommandBuffer(QueueType::TRANSFER_QUEUE, frame_index, thread_index, i); ImageMemoryBarrierSpecification to_transfer = {}; @@ -4610,41 +5654,38 @@ namespace ZEngine::Rendering to_transfer.SourceQueueFamily = m_device->TransferFamilyIndex; to_transfer.DestinationQueueFamily = m_device->TransferFamilyIndex; transfer_cmd->TransitionImageLayout(ImageMemoryBarrier{to_transfer}); - img_buf->Layout = to_transfer.NewLayout; + img_buf->Layout = to_transfer.NewLayout; // A streamed upload is submitted independently from the render timeline. // Keep its staging allocation off the render-timeline ring and retire it // with the producer timeline below. - BufferView transfer_staging = m_device->WriteTextureDataAfterAdmission(transfer_cmd, handle, data, nullptr, false, upload_size); - - ImageMemoryBarrierSpecification release = {}; - release.ImageHandle = buffer_handle; - release.OldLayout = ImageLayout::TRANSFER_DST_OPTIMAL; - release.NewLayout = (img_buf_aspect & VK_IMAGE_ASPECT_DEPTH_BIT) ? ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL : ImageLayout::SHADER_READ_ONLY_OPTIMAL; - release.ImageAspectMask = VkImageAspectFlagBits(img_buf_aspect); - release.SourceAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - release.DestinationAccessMask = VK_ACCESS_NONE; - release.SourceStageMask = VK_PIPELINE_STAGE_TRANSFER_BIT; + transfer_cmd->CopyBufferToImage(staging, img_buf->GetBuffer(), texture->Width, texture->Height, texture->Specification.LayerCount, Specifications::ImageLayoutMap[VALUE_FROM_SPEC_MAP(img_buf->Layout)], 0, texture->Specification.Depth); + + ImageMemoryBarrierSpecification release = {}; + release.ImageHandle = buffer_handle; + release.OldLayout = ImageLayout::TRANSFER_DST_OPTIMAL; + release.NewLayout = (img_buf_aspect & VK_IMAGE_ASPECT_DEPTH_BIT) ? ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL : ImageLayout::SHADER_READ_ONLY_OPTIMAL; + release.ImageAspectMask = VkImageAspectFlagBits(img_buf_aspect); + release.SourceAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + release.DestinationAccessMask = VK_ACCESS_NONE; + release.SourceStageMask = VK_PIPELINE_STAGE_TRANSFER_BIT; // This is the producer half of a queue-family transfer, recorded // on a transfer-only queue. The render graph records the matching // graphics-side acquire with the actual consumer stage. - release.DestinationStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT; - release.LayerCount = texture->Specification.LayerCount; - release.MipLevelCount = texture->Specification.MipLevelCount; - const uint32_t producer_family = m_device->GetQueue(QueueType::TRANSFER_QUEUE).FamilyIndex; - const bool transfers_ownership = producer_family != m_device->GraphicFamilyIndex; - release.SourceQueueFamily = transfers_ownership ? producer_family : VK_QUEUE_FAMILY_IGNORED; - release.DestinationQueueFamily = transfers_ownership ? m_device->GraphicFamilyIndex : VK_QUEUE_FAMILY_IGNORED; + release.DestinationStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT; + release.LayerCount = texture->Specification.LayerCount; + release.MipLevelCount = texture->Specification.MipLevelCount; + const uint32_t producer_family = m_device->GetQueue(QueueType::TRANSFER_QUEUE).FamilyIndex; + const bool transfers_ownership = producer_family != m_device->GraphicFamilyIndex; + release.SourceQueueFamily = transfers_ownership ? producer_family : VK_QUEUE_FAMILY_IGNORED; + release.DestinationQueueFamily = transfers_ownership ? m_device->GraphicFamilyIndex : VK_QUEUE_FAMILY_IGNORED; transfer_cmd->TransitionImageLayout(ImageMemoryBarrier{release}); img_buf->Layout = release.NewLayout; transfer_cmd->End(); - uint64_t transfer_val = m_tex_transfer_next_values[pool_index].fetch_add(1, std::memory_order_acq_rel); - transfer_retire[i] = transfer_val; - if (transfer_staging) - { - m_tex_transfer_staging[pool_index][i] = transfer_staging; - } + uint64_t transfer_val = m_tex_transfer_next_values[pool_index].fetch_add(1, std::memory_order_acq_rel); + retire_values[i] = transfer_val; + m_tex_transfer_staging[pool_index][i] = staging; m_async_uploads.Enqueue({ .Buffer = transfer_cmd, @@ -4662,25 +5703,6 @@ namespace ZEngine::Rendering } else { - auto& retire_values = m_tex_retire_values[pool_index]; - uint32_t i = 0; - for (; i < GEOMETRY_UPLOAD_SLOT; ++i) - { - if (retire_values[i] == 0) - { - break; - } - } - if (i >= GEOMETRY_UPLOAD_SLOT) - { - ZENGINE_CORE_WARN("[RRM] UploadTextureBuffer: no free graphics slot — upload deferred") - return {}; - } - if (!BeginTextureUploadRecording(handle, true)) - { - return {}; - } - auto cmd = m_device->CommandBufferMgr->GetInstantCommandBuffer(QueueType::GRAPHIC_QUEUE, frame_index, thread_index, i); ImageMemoryBarrierSpecification to_transfer = {}; @@ -4699,11 +5721,11 @@ namespace ZEngine::Rendering to_transfer.SourceQueueFamily = m_device->GraphicFamilyIndex; to_transfer.DestinationQueueFamily = m_device->GraphicFamilyIndex; cmd->TransitionImageLayout(ImageMemoryBarrier{to_transfer}); - img_buf->Layout = to_transfer.NewLayout; + img_buf->Layout = to_transfer.NewLayout; // See the dedicated-transfer branch above: the upload submission is not // represented by RenderTimeline, so its staging buffer must not use the ring. - BufferView staging = m_device->WriteTextureDataAfterAdmission(cmd, handle, data, nullptr, false, upload_size); + cmd->CopyBufferToImage(staging, img_buf->GetBuffer(), texture->Width, texture->Height, texture->Specification.LayerCount, Specifications::ImageLayoutMap[VALUE_FROM_SPEC_MAP(img_buf->Layout)], 0, texture->Specification.Depth); ImageMemoryBarrierSpecification to_final = {}; to_final.ImageHandle = buffer_handle; @@ -4721,13 +5743,10 @@ namespace ZEngine::Rendering cmd->TransitionImageLayout(ImageMemoryBarrier{to_final}); cmd->End(); - uint64_t signal_value = m_tex_next_values[pool_index].fetch_add(1, std::memory_order_acq_rel); - retire_values[i] = signal_value; - if (staging) - { - m_tex_retire_staging[pool_index][i] = staging; - } - img_buf->Layout = to_final.NewLayout; + uint64_t signal_value = m_tex_next_values[pool_index].fetch_add(1, std::memory_order_acq_rel); + retire_values[i] = signal_value; + m_tex_retire_staging[pool_index][i] = staging; + img_buf->Layout = to_final.NewLayout; m_async_uploads.Enqueue({ .Buffer = cmd, .Timeline = m_tex_timelines[pool_index], @@ -4881,7 +5900,7 @@ namespace ZEngine::Rendering { const auto handle = deferral.TexHandle; const auto* texture = m_device ? m_device->GlobalTextures.Access(handle) : nullptr; - const bool terminal = !texture || !deferral.Pixels || handle.Index >= TextureVersionCapacity || !texture->BufferSize || !m_device->ImageBufferManager.Access(texture->BufferHandle) || (deferral.ByteSize && deferral.ByteSize > texture->BufferSize) || TextureReleaseRequested(handle) || GetTextureDecodeState(handle) == TextureDecodeState::Failed; + const bool terminal = !texture || !deferral.Pixels || handle.Index >= TextureVersionCapacity || !texture->ContainsBaseMip(deferral.ByteSize ? deferral.ByteSize : texture->BufferSize) || !m_device->ImageBufferManager.Access(texture->BufferHandle) || TextureReleaseRequested(handle) || GetTextureDecodeState(handle) == TextureDecodeState::Failed; if (terminal) { // A late decode is allowed to finish, not to revive a destroyed @@ -5110,6 +6129,7 @@ namespace ZEngine::Rendering { producer.AcquirePending = false; producer.GraphicsUploadOnly = true; + producer.ConsumerDiscarded = true; } } m_streaming_upload_tickets.erase(index); @@ -5117,6 +6137,41 @@ namespace ZEngine::Rendering } } + bool RenderResourceManager::AcknowledgeStreamingUploadAcquire(const Hardwares::StreamingUploadTicket& ticket, Primitives::Semaphore* timeline, uint64_t value, uint32_t consumer_family) + { + if (!ticket.Texture.Valid() || !timeline || !value || consumer_family == VK_QUEUE_FAMILY_IGNORED) + { + return false; + } + for (uint32_t index = 0; index < m_streaming_upload_tickets.size(); ++index) + { + const auto& pending = m_streaming_upload_tickets[index]; + if (pending.Texture != ticket.Texture || pending.CompletionTimeline != ticket.CompletionTimeline || pending.CompletionValue != ticket.CompletionValue || pending.PreReleaseLayout != ticket.PreReleaseLayout || pending.PostReleaseLayout != ticket.PostReleaseLayout || pending.ProducerQueueFamily != ticket.ProducerQueueFamily) + { + continue; + } + if (ticket.Texture.Index < TextureVersionCapacity) + { + std::lock_guard lock(m_texture_version_mutex); + auto& producer = m_texture_producer_completions[ticket.Texture.Index]; + if (producer.Texture != ticket.Texture || !producer.AcquirePending || producer.PendingSubmission || producer.ConsumerTimeline) + { + return false; + } + producer.AcquirePending = false; + producer.ConsumerTimeline = timeline; + producer.ConsumerValue = value; + producer.ConsumerFamily = consumer_family; + producer.ConsumerAcquired = false; + producer.ConsumerDiscarded = false; + producer.GraphicsUploadOnly |= consumer_family == m_device->GraphicFamilyIndex; + } + m_streaming_upload_tickets.erase(index); + return true; + } + return false; + } + void RenderResourceManager::RetireTextureSlots(uint8_t frame_index, uint8_t thread_index) { uint32_t pool_index = (frame_index * m_device->CommandBufferMgr->TotalThreadCount) + thread_index; @@ -5483,6 +6538,11 @@ namespace ZEngine::Rendering ++index; continue; } + if (!TextureConsumerWorkRetired(handle)) + { + ++index; + continue; + } if (const auto* ticket = FindStreamingUploadTicket(handle)) { // An obsolete preview may never be drawn/imported by the graph. diff --git a/ZEngine/ZEngine/Rendering/RenderResourceManager.h b/ZEngine/ZEngine/Rendering/RenderResourceManager.h index ef5d4180e..fcb008159 100644 --- a/ZEngine/ZEngine/Rendering/RenderResourceManager.h +++ b/ZEngine/ZEngine/Rendering/RenderResourceManager.h @@ -1,6 +1,8 @@ #pragma once +#include #include #include +#include #include #include #include @@ -42,6 +44,25 @@ namespace ZEngine::Rendering::Primitives struct Fence; } // namespace ZEngine::Rendering::Primitives +namespace ZEngine::Rendering::Meshes +{ + class CookedMeshStaging; +} + +namespace ZEngine::Managers +{ + class CookedCpuAssetSet; +} +namespace ZEngine::Rendering::Materials +{ + class CookedMaterialVersion; +} +namespace ZEngine::Rendering::Textures +{ + class CookedEnvironmentUpload; + class CookedTextureUpload; +} // namespace ZEngine::Rendering::Textures + namespace ZEngine::Rendering { // RenderResourceManager — single authority over GPU buffer/image lifetime. @@ -64,6 +85,52 @@ namespace ZEngine::Rendering class RenderResourceManager { public: + /// Process-test fault injection: reject one batch containing a cooked mesh. + /// The render thread consumes this only immediately before queue submission. + void ArmCookedBatchRejectionProbe() noexcept + { + m_reject_cooked_batch_probe.store(true, std::memory_order_release); + } + /// Exclusive owner of unpublished host staging. The device must outlive + /// it; handoffs between threads require external synchronization. The + /// source bytes passed to Append must remain immutable until consumed. + /// This owner does not retain or identify the source asset; the caller + /// must retain its exact version and bind it to the intended texture. + class UploadStaging final + { + public: + static constexpr size_t MaxSliceBytes = 64 * 1024; + UploadStaging() = default; + ~UploadStaging(); + UploadStaging(const UploadStaging&) = delete; + UploadStaging& operator=(const UploadStaging&) = delete; + UploadStaging(UploadStaging&&) noexcept; + UploadStaging& operator=(UploadStaging&&) noexcept; + + /// Writes at most MaxSliceBytes. Failure does not advance Written. + [[nodiscard]] bool Append(std::span bytes, size_t* consumed, VkResult* status = nullptr); + void Reset() noexcept; + [[nodiscard]] bool Ready() const noexcept + { + return m_device && m_written == m_buffer.Size; + } + [[nodiscard]] VkDeviceSize Written() const noexcept + { + return m_written; + } + [[nodiscard]] VkDeviceSize Size() const noexcept + { + return m_buffer.Size; + } + + private: + friend class RenderResourceManager; + friend struct ::RRMUploadBatchTestHelper; + Hardwares::VulkanDevice* m_device = nullptr; + Core::Memory::BufferView m_buffer = {}; + VkDeviceSize m_written = 0; + }; + /// Immutable draw layout captured from the same CPU mesh as the upload. /// Material UUIDs are identities, not retained material/texture versions. struct MeshDrawMetadata @@ -222,44 +289,142 @@ namespace ZEngine::Rendering ResourcesUnavailable, }; - TextureVersionPin PinTextureVersion(Textures::TextureHandle handle); + TextureVersionPin PinTextureVersion(Textures::TextureHandle handle); /// Retains a graph-owned image identity and its shared allocation, not /// its contents. Graph barriers still order writes and alias reuse. - TextureVersionPin PinGraphTextureVersion(Textures::TextureHandle handle); + TextureVersionPin PinGraphTextureVersion(Textures::TextureHandle handle); /// Extend an existing exact texture lease into accepted-submission /// retirement. Does not reacquire a UUID or reopen sealed admission. /// Output is unchanged on refusal; the render owner calls this before /// recording a consumer, including pass-local sampled descriptors. - bool GetFrameTextureVersion(const TextureVersionPin& retained, TextureVersionSnapshot* snapshot); - TextureVersionPin RetainTextureVersion(const TextureVersionPin& retained); + bool GetFrameTextureVersion(const TextureVersionPin& retained, TextureVersionSnapshot* snapshot); + TextureVersionPin RetainTextureVersion(const TextureVersionPin& retained); + /// Render-owner preparation for an already submitted upload. Retains its + /// exact image and observes producer completion and bindless publication. + /// Ready here does not prove a consumer acquire or a complete scene. + [[nodiscard]] SceneTextureDescriptorState PrepareSubmittedTextureVersion(Textures::TextureHandle handle, TextureVersionPin* retained, uint64_t* publication); + [[nodiscard]] bool IsSubmittedTextureDescriptorCurrent(const TextureVersionPin& retained, uint64_t publication); + /// Observes the exact accepted consumer-acquire submission. Ready means + /// completion on the graphics family; an erased ticket alone is not proof. + [[nodiscard]] SceneTextureDescriptorState PollSubmittedTextureAcquire(const TextureVersionPin& retained); + static constexpr uint32_t CookedMeshUploadCapacity = 16; + struct CookedMeshUploadId + { + uint32_t Slot = UINT32_MAX; + uint64_t Serial = 0; + }; + enum class CookedMeshAdmission : uint8_t + { + Recorded, + Retry, + Failed + }; + enum class CookedMeshUploadState : uint8_t + { + Invalid, + Pending, + Ready, + Failed, + Cancelled + }; + struct CookedMeshUploadResult + { + CookedMeshUploadState State = CookedMeshUploadState::Invalid; + bool CleanupPending = false; + Cook::CookAssetIdentity Identity = {}; + Core::Hash::Digest256 Digest = {}; + }; + /// Render-owner lease for a completed cooked upload. Identity and + /// digest travel with the exact geometry; the manager outlives it. + class CookedMeshVersion final + { + public: + CookedMeshVersion() = default; + ~CookedMeshVersion(); + CookedMeshVersion(const CookedMeshVersion&) = delete; + CookedMeshVersion& operator=(const CookedMeshVersion&) = delete; + CookedMeshVersion(CookedMeshVersion&& other) noexcept; + CookedMeshVersion& operator=(CookedMeshVersion&& other) noexcept; + explicit operator bool() const noexcept + { + return m_owner != nullptr; + } + const Cook::CookAssetIdentity& Identity() const noexcept + { + return m_identity; + } + const Core::Hash::Digest256& Digest() const noexcept + { + return m_digest; + } + const MeshGeometrySnapshot* Geometry() const + { + return m_pin.Get(); + } + void Reset(); + + private: + friend class RenderResourceManager; + RenderResourceManager* m_owner = nullptr; + BufferHandle m_handle = {}; + MeshGeometryPin m_pin = {}; + Cook::CookAssetIdentity m_identity = {}; + Core::Hash::Digest256 m_digest = {}; + }; + /// Render-owner only. Records a staged cooked mesh without waiting for + /// a batch slot. The bounded metadata slab must outlive every resulting + /// mesh version and pin. Retry/Failed preserve the staged candidate. + /// A batch submission failure is terminal for cooked mesh admission; + /// only capacity or a busy slot produces Retry. + [[nodiscard]] CookedMeshAdmission RecordCookedMeshUpload(uint8_t frame_index, Meshes::CookedMeshStaging& staging, Core::Memory::TLSFSlab& metadata_slab, CookedMeshUploadId* output); + /// Recorded work stays pinned until acceptance and GPU completion. A + /// cancelled outcome can precede cleanup; keep polling until it drains. + [[nodiscard]] CookedMeshUploadResult PollCookedMeshUpload(CookedMeshUploadId id); + [[nodiscard]] bool CancelCookedMeshUpload(CookedMeshUploadId id); + /// Transfers the ready pin and handle to the caller. The caller must + /// eventually Release(handle) and retain the pin through consumers. + [[nodiscard]] bool TakeCookedMeshUpload(CookedMeshUploadId id, BufferHandle* handle, MeshGeometryPin* pin); + /// Takes a completed upload without dropping its cooked identity. + [[nodiscard]] bool TakeCookedMeshUpload(CookedMeshUploadId id, CookedMeshVersion* output); + /// Release a terminal result after cleanup, or discard a ready result. + [[nodiscard]] bool ReleaseCookedMeshUpload(CookedMeshUploadId id); + /// Joined-owner shutdown only: the device is idle or terminally lost and + /// no recorded work can subsequently be submitted. + [[nodiscard]] bool AbandonCookedMeshUploadAfterDeviceIdle(CookedMeshUploadId id); /// RequiredSceneBindings validates authored texture identities under the /// cache lock and rejects debug fallbacks before retaining any GPU map. - MaterialVersionPin PinMaterialVersion(const uuids::uuid& uuid, MaterialVersionPolicy policy = MaterialVersionPolicy::ExistingBindings, MaterialVersionError* error = nullptr); - static constexpr uint32_t FrameMaterialCapacity = 4096; + MaterialVersionPin PinMaterialVersion(const uuids::uuid& uuid, MaterialVersionPolicy policy = MaterialVersionPolicy::ExistingBindings, MaterialVersionError* error = nullptr); + /// Render-owner preparation from explicit texture pins, in shader map + /// order. Each nonempty map must match a pin from this manager. Ready + /// requires the exact graphics acquire and current descriptor versions; + /// Pending/Failed leave output unchanged. No asset registry is consulted. + [[nodiscard]] SceneTextureDescriptorState PrepareExplicitMaterialVersion(const Meshes::MeshMaterial& material, const TextureVersionPin* const (&textures)[5], MaterialVersionPin* output); + static constexpr uint32_t FrameMaterialCapacity = 4096; /// Render-thread draw admission. The returned table index and copied /// shader values refer to one retained version, shared across draws/frames. /// Legacy scene callers explicitly opt into the cache's fallback material. - bool GetFrameMaterialVersion(const uuids::uuid& uuid, uint32_t* index, MaterialVersionSnapshot* snapshot, bool use_fallback = false); - bool GetFrameMaterialVersion(const Meshes::MeshMaterial& material, uint32_t* index, MaterialVersionSnapshot* snapshot); + bool GetFrameMaterialVersion(const uuids::uuid& uuid, uint32_t* index, MaterialVersionSnapshot* snapshot, bool use_fallback = false); + bool GetFrameMaterialVersion(const Meshes::MeshMaterial& material, uint32_t* index, MaterialVersionSnapshot* snapshot); /// Extend an existing lease into automatic frame retirement without /// reacquiring the mutable cache or reopening sealed texture admission. - bool GetFrameMaterialVersion(const MaterialVersionPin& retained, uint32_t* index, MaterialVersionSnapshot* snapshot); + bool GetFrameMaterialVersion(const MaterialVersionPin& retained, uint32_t* index, MaterialVersionSnapshot* snapshot); /// Upload only admitted immutable entries, never the mutable asset cache. - void UploadFrameMaterials(Core::Memory::BufferView& destination); + void UploadFrameMaterials(Core::Memory::BufferView& destination); /// Device mutation/retirement guards; callable by producer threads. - bool IsTextureVersionPinned(Textures::TextureHandle handle) const; - bool CanWriteTextureVersion(Textures::TextureHandle handle) const; + bool IsTextureVersionPinned(Textures::TextureHandle handle) const; + bool CanWriteTextureVersion(Textures::TextureHandle handle) const; /// Managed writers observe producer completion, require acknowledged /// ownership and invalidate previous admission proof before changing bytes. - bool BeginTextureVersionWrite(Textures::TextureHandle handle); + bool BeginTextureVersionWrite(Textures::TextureHandle handle); /// force_defer preserves retirement when the bounded native queue refuses admission. - bool RetainTextureForRetirement(Textures::TextureHandle handle, bool force_defer = false); + bool RetainTextureForRetirement(Textures::TextureHandle handle, bool force_defer = false); - static constexpr uint32_t SceneResourcePreparationCapacity = 2; - static constexpr uint32_t SceneGeometryMeshCapacity = 4096; - static constexpr uint32_t SceneResourceSliceOperations = 16; - static constexpr uint32_t SceneMaterialCapacity = 4096; - static constexpr uint32_t SceneDrawCapacity = Scenes::SceneData::MAX_DRAW_COMMANDS; + static constexpr uint32_t SceneResourcePreparationCapacity = 2; + static constexpr uint32_t SceneGeometryMeshCapacity = 4096; + static constexpr uint32_t SceneResourceSliceOperations = 16; + static constexpr uint32_t SceneMaterialCapacity = 4096; + static constexpr uint32_t SceneCookedTextureCapacity = TextureVersionCapacity; + static constexpr uint32_t SceneDrawCapacity = Scenes::SceneData::MAX_DRAW_COMMANDS; /// Main-producer input copied into fixed request storage at admission. /// The span is never borrowed by render-owner preparation. @@ -360,6 +525,9 @@ namespace ZEngine::Rendering uint64_t ExpectedGeneration = 0; SceneEnvironmentPreparationState State = SceneEnvironmentPreparationState::NotRequested; bool Cancelled = false; + Textures::TextureHandle CookedSource = {}; + uint32_t CookedFaceWidth = 0; + uint64_t CookedSourceHash = 0; }; struct SceneResourcePreparationResult { @@ -370,6 +538,9 @@ namespace ZEngine::Rendering SceneResourcePreparationError Error = SceneResourcePreparationError::None; MaterialVersionError MaterialError = MaterialVersionError::None; ScenePreparationContent Content = ScenePreparationContent::GeometryOnly; + bool Cooked = false; + Core::Hash::Digest256 CookedManifest = {}; + uuids::uuid CookedScene = {}; // Published only at GeometryReady/ResourcesRetained or Failed. uint32_t MaterialCount = 0; bool FrameRecorded = false; @@ -395,7 +566,23 @@ namespace ZEngine::Rendering /// from retained metadata/material reservations under the same slice /// budget. Draw overflow/invalid ranges fail before resource readiness. /// GeometryOnly preserves the narrow upload-readiness contract. - bool PrepareSceneResources(ECS::SceneRequestId request, uint64_t expected_generation, std::span meshes, SceneResourcePreparationId* out, ScenePreparationContent content = ScenePreparationContent::GeometryOnly, const SceneEnvironmentInput* environment = nullptr, const SceneInitialFrameInput* initial_frame = nullptr, bool defer_instances = false); + bool PrepareSceneResources(ECS::SceneRequestId request, uint64_t expected_generation, std::span meshes, SceneResourcePreparationId* out, ScenePreparationContent content = ScenePreparationContent::GeometryOnly, const SceneEnvironmentInput* environment = nullptr, const SceneInitialFrameInput* initial_frame = nullptr, bool defer_instances = false, const Managers::CookedCpuAssetSet* cooked = nullptr); + [[nodiscard]] bool SceneResourceAdmissionSealed() const noexcept + { + return m_scene_resources_sealed.load(std::memory_order_acquire); + } + /// Render-owner collection, serialized with admission, instance copying, + /// cancellation and release. Each call retains one verified version; + /// Finish queues the complete candidate for ordinary sliced preparation. + [[nodiscard]] bool AppendCookedSceneMesh(SceneResourcePreparationId, const Managers::CookedCpuAssetSet&, const CookedMeshVersion&); + [[nodiscard]] bool AppendCookedSceneMaterial(SceneResourcePreparationId, const Managers::CookedCpuAssetSet&, const Materials::CookedMaterialVersion&); + /// On success transfers the uploaded texture's ownership to the scene. + /// Inputs must arrive once each in ascending UUID order. + [[nodiscard]] bool AppendCookedSceneTexture(SceneResourcePreparationId, const Managers::CookedCpuAssetSet&, Textures::CookedTextureUpload&); + /// On success transfers destination-texture ownership to the request. + /// Refusal preserves caller ownership; retry after upload submission. + [[nodiscard]] bool AppendCookedSceneEnvironment(SceneResourcePreparationId, const Managers::CookedCpuAssetSet&, const Textures::CookedEnvironmentUpload&); + [[nodiscard]] bool FinishCookedSceneResources(SceneResourcePreparationId); /// Producer-thread only, serialized with admission/release. Deferred input /// is copied in batches of at most 64; no producer pointer is retained. /// Rendering cannot observe partial input. Invalid batches fail the request. @@ -429,6 +616,8 @@ namespace ZEngine::Rendering /// Activate requires both the render-owner claim and a retained frame. SceneResourceLease ClaimSceneEnvironmentPreparation(SceneEnvironmentPreparation* out); bool GetSceneEnvironmentPreparation(SceneResourcePreparationId id, SceneEnvironmentPreparation* out) const; + /// Render owner transfers the retained source to the sky scheduler. + [[nodiscard]] bool TakeCookedSceneEnvironment(SceneResourcePreparationId, Textures::TextureHandle* out); bool CompleteSceneEnvironmentPreparation(SceneResourcePreparationId id, bool success, SceneResourcePreparationError error = SceneResourcePreparationError::RequiredEnvironment); bool ActivateSceneEnvironmentPreparation(SceneResourcePreparationId id); @@ -462,24 +651,24 @@ namespace ZEngine::Rendering /// @param registry The asset registry to subscribe to; must outlive this RRM instance. /// @param upload_arena Owns bounded CPU decode storage and must outlive this /// manager and the worker-pool registrations. - void Initialize(Hardwares::VulkanDevice* device, Core::VFS::AssetRegistry* registry, Core::Memory::ArenaAllocator* upload_arena); + void Initialize(Hardwares::VulkanDevice* device, Core::VFS::AssetRegistry* registry, Core::Memory::ArenaAllocator* upload_arena); /// @brief Drain in-flight GPU work and release all GPU resources. /// @details Calls vkQueueWaitAll, shuts down texture timelines, and frees the /// global geometry buffers and all generic buffer slots. Must be called /// from the main thread before VulkanDevice teardown. - void Shutdown(); + void Shutdown(); /// @brief Per-frame render-thread entry point. /// @details Flushes pending uploads, swaps, and texture reloads/releases queued /// from other threads since the previous BeginFrame. Called by AppRenderPipeline. /// @param frame_index Current swapchain frame index (0 .. FRAMES_IN_FLIGHT-1). - void BeginFrame(uint32_t frame_index); + void BeginFrame(uint32_t frame_index); /// @brief Closes the current deferred upload batch, if anything joined it. /// @details Called by RenderGraph after its declaration phase and by /// AppRenderPipeline::EndFrame as an idempotent fallback. - void EndFrame(); + void EndFrame(); /// @brief Ingest a texture file, uploading (existing invalid) or reloading it in /// place (existing valid) under the same handle. @@ -487,25 +676,25 @@ namespace ZEngine::Rendering /// @param absolute_path Absolute filesystem path to the image file. /// @param existing Current handle when reimporting; invalid for first ingest. /// @return The texture's handle — same as existing when reimporting. - Rendering::Textures::TextureHandle IngestTexture(const uuids::uuid& uuid, const char* absolute_path, Rendering::Textures::TextureHandle existing = {}); + Rendering::Textures::TextureHandle IngestTexture(const uuids::uuid& uuid, const char* absolute_path, Rendering::Textures::TextureHandle existing = {}); /// @brief Schedule a hot-reload reimport for a texture, deduped by UUID. /// @details Thread-safe; applied by the next FlushPendingTextureReloads. - void ScheduleTextureReload(const uuids::uuid& uuid); + void ScheduleTextureReload(const uuids::uuid& uuid); /// Render-thread readiness guard for material-preview rendering. A valid /// bindless handle alone does not mean its asynchronous source decode ran. - bool HasPendingTextureUploads(); + bool HasPendingTextureUploads(); /// @brief Release a texture: patch referencing materials to the sentinel, then /// timeline-gate the GPU-side teardown. /// @details Thread-safe. The CPU-side patch always lands before the bindless slot /// can be reused — see Device->DestroyTexture for the GPU-side gate. - void ReleaseTexture(const uuids::uuid& uuid); + void ReleaseTexture(const uuids::uuid& uuid); /// @brief Look up a texture by handle. /// @return Pointer to the Texture, or nullptr if the handle is invalid or stale. - const Rendering::Textures::Texture* GetTexture(const Rendering::Textures::TextureHandle& handle) const; + const Rendering::Textures::Texture* GetTexture(const Rendering::Textures::TextureHandle& handle) const; /// @brief Upload raw RGBA pixel data to an existing TextureHandle via the timeline path. /// @details Records a staging copy into an instant command buffer and enqueues the @@ -515,14 +704,24 @@ namespace ZEngine::Rendering /// @param thread_index Thread index within the pool. /// @param handle Pre-allocated TextureHandle whose VkImage will receive the data. /// @param data RGBA pixel data; must remain valid until SubmitAsyncUploads runs. + /// @param data_size Available bytes, or zero when the caller guarantees + /// that the base mip is present. Only mip zero is copied. /// @return The same handle on success; invalid handle if no free upload slot. - Rendering::Textures::TextureHandle UploadTextureBuffer(uint8_t frame_index, uint8_t thread_index, const Rendering::Textures::TextureHandle& handle, unsigned char* data, size_t data_size = 0); + Rendering::Textures::TextureHandle UploadTextureBuffer(uint8_t frame_index, uint8_t thread_index, const Rendering::Textures::TextureHandle& handle, unsigned char* data, size_t data_size = 0); + + /// Render-owner allocation of bounded unpublished staging. Failure preserves output; the + /// allocation is charged until Reset or transfer to the upload queue. + [[nodiscard]] bool PrepareUploadStaging(VkDeviceSize size, UploadStaging* output, VkResult* status = nullptr); + /// Render-owner only. Incomplete or incompatible staging is refused. + /// Refusal preserves the staging for retry; success transfers ownership + /// to the producer timeline. This is not GPU/descriptor readiness. + [[nodiscard]] Rendering::Textures::TextureHandle UploadPreparedTextureBuffer(uint8_t frame_index, uint8_t thread_index, const Rendering::Textures::TextureHandle& handle, UploadStaging& staging); // Upload the ZUI font atlas synchronously using m_upload_cmd_mgr/m_sync_upload_fence. // Blocks until the GPU copy is complete so the texture is ready before the first // frame renders. Caller must enqueue the returned handle to // TextureHandleToUpdates for bindless descriptor registration. - Rendering::Textures::TextureHandle UploadFontAtlas(unsigned char* pixels, uint32_t width, uint32_t height); + Rendering::Textures::TextureHandle UploadFontAtlas(unsigned char* pixels, uint32_t width, uint32_t height); /// @brief Decode a texture file on the thread pool and upload it, async. /// @details When existing is valid, reconstructs that handle in place instead of @@ -534,12 +733,12 @@ namespace ZEngine::Rendering /// @param existing Handle to reconstruct in place; invalid to allocate a new one. /// @param track_decode When true, retain the decode result for the render-thread caller. /// @return A valid TextureHandle that will become readable once the upload drains. - Rendering::Textures::TextureHandle SubmitTextureFile(const char* filename, Rendering::Textures::TextureHandle existing = {}, bool track_decode = false); + Rendering::Textures::TextureHandle SubmitTextureFile(const char* filename, Rendering::Textures::TextureHandle existing = {}, bool track_decode = false); /// @brief Decode a raster source into a bounded square texture for editor previews. /// @details The source is aspect-fit into a transparent square on a worker and then /// follows the normal asynchronous upload and bindless-registration path. - static constexpr uint32_t ThumbnailTextureSlotReserve = 16; + static constexpr uint32_t ThumbnailTextureSlotReserve = 16; struct ThumbnailCacheIdentity { uuids::uuid AssetUUID = {}; @@ -646,6 +845,10 @@ namespace ZEngine::Rendering /// @details Render-thread only. The ticket is removed after its acquire barrier /// has been recorded into the consuming graph submission. void AcknowledgeStreamingUploadTicket(const Hardwares::StreamingUploadTicket& ticket); + /// Called by the graph only after the acquire-bearing submission is + /// accepted. Its timeline must remain live until ForgetResourceTimeline + /// succeeds. Cancellation/discard must use the non-proof method above. + [[nodiscard]] bool AcknowledgeStreamingUploadAcquire(const Hardwares::StreamingUploadTicket& ticket, Primitives::Semaphore* timeline, uint64_t value, uint32_t consumer_family); /// @brief Retire command buffers whose timeline fence has been signalled. /// @details Frees staging buffers associated with completed texture uploads for the @@ -902,6 +1105,19 @@ namespace ZEngine::Rendering uint64_t RetireValues[ResourceTimelineCount] = {}; }; + struct CookedMeshUploadRecord + { + CookedMeshUploadState State = CookedMeshUploadState::Invalid; + uint64_t Serial = 0; + uint64_t SignalValue = 0; + bool CleanupPending = false; + bool CancelRequested = false; + Cook::CookAssetIdentity Identity = {}; + Core::Hash::Digest256 Digest = {}; + BufferHandle Handle = {}; + MeshGeometryPin Pin = {}; + }; + enum class SceneResourcePhase : uint8_t { Free, @@ -940,13 +1156,28 @@ namespace ZEngine::Rendering std::atomic Phase{SceneResourcePhase::Free}; std::atomic Readers{0}; std::atomic FrameRecorded{false}; - SceneResourcePreparationId Id = {}; - uint64_t ExpectedGeneration = 0; - uint32_t MeshCount = 0; - uuids::uuid Meshes[SceneGeometryMeshCapacity] = {}; - MeshGeometryPin Pins[SceneGeometryMeshCapacity] = {}; - ScenePreparationContent Content = ScenePreparationContent::GeometryOnly; - SceneEnvironmentInput EnvironmentInput = {}; + SceneResourcePreparationId Id = {}; + uint64_t ExpectedGeneration = 0; + uint32_t MeshCount = 0; + uuids::uuid Meshes[SceneGeometryMeshCapacity] = {}; + MeshGeometryPin Pins[SceneGeometryMeshCapacity] = {}; + bool Cooked = false; + Core::Hash::Digest256 CookedManifest = {}; + uuids::uuid CookedScene = {}; + uint32_t CookedMaterialCount = 0; + uuids::uuid CookedMaterials[SceneMaterialCapacity] = {}; + MaterialVersionPin CookedMaterialPins[SceneMaterialCapacity] = {}; + uint32_t CookedTextureCount = 0; + uuids::uuid CookedTextures[SceneCookedTextureCapacity] = {}; + Textures::TextureHandle CookedTextureHandles[SceneCookedTextureCapacity] = {}; + TextureVersionPin CookedTexturePins[SceneCookedTextureCapacity] = {}; + TextureVersionPin CookedEnvironmentPin = {}; + Textures::TextureHandle CookedEnvironment = {}; + uint32_t CookedEnvironmentFace = 0; + uint64_t CookedEnvironmentHash = 0; + bool CookedEnvironmentOwned = false; + ScenePreparationContent Content = ScenePreparationContent::GeometryOnly; + SceneEnvironmentInput EnvironmentInput = {}; std::atomic Environment{SceneEnvironmentPreparationState::NotRequested}; std::atomic EnvironmentError{SceneResourcePreparationError::None}; // Render-owner-only retention/index/progress; never exposed while @@ -1158,6 +1389,7 @@ namespace ZEngine::Rendering bool BeginTextureUploadRecording(Textures::TextureHandle handle, bool graphics_upload); QueueType TextureUploadQueue(Textures::TextureHandle handle, Specifications::ImageLayout layout) const; bool TextureProducerComplete(Textures::TextureHandle handle, bool* failed = nullptr); + bool TextureConsumerWorkRetired(Textures::TextureHandle handle, bool* failed = nullptr); static void OnStreamingUploadSubmitted(void* context, const Hardwares::StreamingUploadTicket& ticket); static void OnStreamingUploadRejected(void* context, const Hardwares::StreamingUploadTicket& ticket); void RejectTextureUpload(const Hardwares::StreamingUploadTicket& ticket); @@ -1165,10 +1397,15 @@ namespace ZEngine::Rendering void BeginBatchUpload(uint8_t frame_index); void EndBatchUpload(); + void DiscardUnsubmittedBatchStaging(); /// @brief Reuse this slot's open batch, or submit the previous slot's batch /// before opening a new one. Every upload joins through this method. void EnsureBatchOpen(uint8_t frame_index); + CookedMeshAdmission TryEnsureCookedBatchOpen(uint8_t frame_index); + void PollCookedMeshUploads(); + void ReleaseCookedMeshResources(CookedMeshUploadRecord& record); + void ReleaseCookedMeshUploadsAfterIdle(); /// @brief Free frame-index batch stagings once m_batch_timeline reaches LastSignal. /// Called once per frame from BeginFrame. @@ -1199,7 +1436,7 @@ namespace ZEngine::Rendering void AdmitTextureVersionRetirements(); void ReplaceMeshGeometry(MeshSlot& geometry, const MeshSlot& replacement); void ReleaseMeshGeometryPin(uint32_t record, uint64_t version); - void TickSceneResourcePreparations(uint64_t completed, bool upload_failed); + void TickSceneResourcePreparations(uint64_t completed, bool upload_failed, bool invalidate_ready = true); static uint32_t PreparedMaterialBucket(const SceneResourcePreparation& record, const uuids::uuid& material); const MeshGeometryPin* FindPreparedSceneGeometry(SceneResourcePreparationId id, const uuids::uuid& mesh) const; const MaterialVersionPin* FindPreparedSceneMaterial(SceneResourcePreparationId id, const uuids::uuid& material) const; @@ -1277,7 +1514,11 @@ namespace ZEngine::Rendering Core::Containers::Array m_batch_frames = {}; uint64_t m_batch_next_value = 0; uint64_t m_batch_accepted_value = 0; + uint32_t m_batch_staging_base_count = 0; bool m_batch_submission_failed = false; + std::atomic m_reject_cooked_batch_probe = false; + CookedMeshUploadRecord m_cooked_mesh_uploads[CookedMeshUploadCapacity] = {}; + uint64_t m_cooked_mesh_upload_serial = 0; SceneResourcePreparation m_scene_resources[SceneResourcePreparationCapacity] = {}; uint64_t m_scene_resources_sequence = 0; uint32_t m_scene_resources_turn = 0; @@ -1452,6 +1693,11 @@ namespace ZEngine::Rendering bool PendingSubmission = false; bool AcquirePending = false; bool GraphicsUploadOnly = false; + Primitives::Semaphore* ConsumerTimeline = nullptr; + uint64_t ConsumerValue = 0; + uint32_t ConsumerFamily = VK_QUEUE_FAMILY_IGNORED; + bool ConsumerAcquired = false; + bool ConsumerDiscarded = false; }; TextureProducerCompletion m_texture_producer_completions[TextureVersionCapacity] = {}; uint32_t m_tex_total_cmd_count = 0; diff --git a/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.cpp b/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.cpp index 8c1c94b64..7408479ba 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.cpp +++ b/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.cpp @@ -354,6 +354,14 @@ namespace ZEngine::Rendering::Renderers } } + void GraphicRenderer::RequestSceneColorCapture(RGReadbackFn complete, RGReadbackCancelFn cancelled, void* context) + { + if (m_tone_mapping_pass) + m_tone_mapping_pass->RequestCapture({complete, cancelled, context}); + else if (cancelled) + cancelled(context); + } + void GraphicRenderer::RequestObjectIdPick(const ObjectIdPickRequest& request) { if (m_object_id_pass) @@ -375,6 +383,8 @@ namespace ZEngine::Rendering::Renderers ZENGINE_VALIDATE_ASSERT(frame_index < Scenes::SceneData::MAX_FRAMES_IN_FLIGHT, "Invalid scene-buffer frame index") if (initial_frame && initial_frame->View.Captured && !initial_frame->CameraBuffer) { + if (m_tone_mapping_pass) + m_tone_mapping_pass->CancelPendingCapture(); return nullptr; } const auto* transforms = initial_frame && initial_frame->TransformBuffer ? initial_frame->TransformBuffer : &RenderSceneData->TransformBuffers[frame_index]; @@ -414,7 +424,8 @@ namespace ZEngine::Rendering::Renderers m_asset_preview_pass->BeginFrame(); Hardwares::CommandBuffer* const output = RenderGraph->Execute(cb); - RenderSceneData->ActiveCameraBuffer = nullptr; + m_tone_mapping_pass->CancelPendingCapture(); + RenderSceneData->ActiveCameraBuffer = nullptr; m_asset_preview_pass->EndFrame(); FinishSkyFrameRecording(output != nullptr); if (output) @@ -527,7 +538,7 @@ namespace ZEngine::Rendering::Renderers m_memory_statistics_sequence.value.fetch_add(1, std::memory_order_release); } - bool GraphicRenderer::SubmitSkySnapshot(const Scenes::SkyConfig& sky, const Scenes::SkyCelestialLight& celestial_light, uint64_t revision, Scenes::SkyEnvironmentPublication publication) + bool GraphicRenderer::SubmitSkySnapshot(const Scenes::SkyConfig& sky, const Scenes::SkyCelestialLight& celestial_light, uint64_t revision, Scenes::SkyEnvironmentPublication publication, const RenderResourceManager::SceneEnvironmentPreparation* preparation) { ZENGINE_PROFILE_SCOPE("SceneLoad.Render.SkyRequest"); const EnvironmentLightingBakeSettings bake_settings = Device ? Device->EnvironmentLightingBakeSettings : ResolveEnvironmentLightingQuality(EnvironmentLightingQualityTier::Standard); @@ -535,7 +546,12 @@ namespace ZEngine::Rendering::Renderers bool hdri_artifact_ready = !sky.IsHDRI(); if (sky.IsHDRI()) { - if (auto* const asset_manager = ZEngine::Managers::AssetManager::Instance(); asset_manager && asset_manager->Registry) + if (preparation && preparation->CookedSource.Valid() && preparation->Input.Sky.IsHDRI() && preparation->Input.Sky.EnvironmentMap == sky.EnvironmentMap) + { + hdri_source_hash = preparation->CookedSourceHash; + hdri_artifact_ready = true; + } + else if (auto* const asset_manager = ZEngine::Managers::AssetManager::Instance(); asset_manager && asset_manager->Registry) { std::lock_guard lock(asset_manager->IngestMutex); if (const auto* const environment = asset_manager->Registry->FindByUUID(sky.EnvironmentMap)) @@ -730,7 +746,7 @@ namespace ZEngine::Rendering::Renderers } m_scene_pipeline_state = RGScenePipelinePreparationState::Preparing; m_scene_pipeline_active = true; - if (!SubmitSkySnapshot(preparation.Input.Sky, preparation.Input.CelestialLight, m_scene_sky_revision, Scenes::SkyEnvironmentPublication::Explicit)) + if (!SubmitSkySnapshot(preparation.Input.Sky, preparation.Input.CelestialLight, m_scene_sky_revision, Scenes::SkyEnvironmentPublication::Explicit, &preparation)) { rrm->CompleteSceneEnvironmentPreparation(m_scene_sky_id, false); ReleaseSceneEnvironmentPreparation(); @@ -828,14 +844,16 @@ namespace ZEngine::Rendering::Renderers { ZENGINE_PROFILE_SCOPE("SceneLoad.Render.EnvironmentActivation"); PollSceneEnvironmentPreparations(); + RenderResourceManager::SceneEnvironmentPreparation preparation = {}; + const RenderResourceManager::SceneEnvironmentPreparation* retained_preparation = nullptr; if (resources) { - RenderResourceManager::SceneEnvironmentPreparation preparation = {}; - auto* const rrm = Device && Device->RRM ? static_cast(Device->RRM) : nullptr; + auto* const rrm = Device && Device->RRM ? static_cast(Device->RRM) : nullptr; if (!rrm || !rrm->GetSceneEnvironmentPreparation(resources, &preparation) || preparation.ExpectedGeneration + 1 != scene_generation) { return false; } + retained_preparation = &preparation; if (preparation.State == RenderResourceManager::SceneEnvironmentPreparationState::Ready) { if (preparation.Cancelled || resources != m_scene_sky_id || !m_scene_texture_sources_ready || !m_sky_environment.GetPreparedSnapshot(m_scene_sky_revision) || !m_scene_pipeline_preparation->ActivateSceneDescriptors() || !rrm->ActivateSceneEnvironmentPreparation(resources)) @@ -862,6 +880,15 @@ namespace ZEngine::Rendering::Renderers { return false; } + // A cooked environment is immutable for its scene generation. + // Cancelling a newer prepared candidate must keep the already + // published sky, rather than re-resolving the outgoing HDRI through + // the authoring registry or starting another bake. + RenderResourceManager::SceneResourcePreparationResult resource_result = {}; + if (preparation.State == RenderResourceManager::SceneEnvironmentPreparationState::Activated && m_presented_scene_generation == scene_generation && m_presented_sky_revision == revision && rrm->PollSceneResourcePreparation(resources, &resource_result) && resource_result.Cooked) + { + return true; + } } // Coalesced authoring edits cannot replace the prepared input before // its first successful recording, including retries after graph rejection. @@ -889,7 +916,7 @@ namespace ZEngine::Rendering::Renderers m_presented_scene_generation = scene_generation; m_presented_sky_revision = revision; } - if (SubmitSkySnapshot(sky, celestial_light, m_presented_render_sky_revision, Scenes::SkyEnvironmentPublication::Immediate)) + if (SubmitSkySnapshot(sky, celestial_light, m_presented_render_sky_revision, Scenes::SkyEnvironmentPublication::Immediate, retained_preparation)) { StartPendingSkyBake(); } @@ -1060,8 +1087,31 @@ namespace ZEngine::Rendering::Renderers return; } + auto* const rrm = Device && Device->RRM ? static_cast(Device->RRM) : nullptr; + RenderResourceManager::SceneEnvironmentPreparation cooked = {}; + if (rrm && m_scene_sky_lease && request.Revision == m_scene_sky_revision && rrm->GetSceneEnvironmentPreparation(m_scene_sky_id, &cooked) && cooked.CookedSource.Valid()) + { + cstring reason = nullptr; + if (cooked.Cancelled || !SupportsHDRISourceResources(cooked.CookedFaceWidth, &reason) || !m_sky_environment.ReserveActiveBakeMemory(request.Revision, EstimateHDRIBakeBytes(request.BakeSettings, cooked.CookedFaceWidth))) + { + m_sky_environment.CompleteBake(request.Revision, {}, false); + return; + } + Textures::TextureHandle source = {}; + if (!rrm->TakeCookedSceneEnvironment(m_scene_sky_id, &source)) + { + m_sky_environment.CompleteBake(request.Revision, {}, false); + return; + } + if (!m_sky_environment.AttachBakeResource(request.Revision, source)) + { + m_sky_environment.CompleteBake(request.Revision, source, false); + DiscardSkyTexture(source); + } + return; + } + auto* const asset_manager = ZEngine::Managers::AssetManager::Instance(); - auto* const rrm = Device && Device->RRM ? static_cast(Device->RRM) : nullptr; if (!asset_manager || !asset_manager->Registry || !rrm) { ZENGINE_CORE_ERROR("[SkyEnvironment] Revision {} is using the fallback: asset services are unavailable", request.Revision) @@ -1180,13 +1230,36 @@ namespace ZEngine::Rendering::Renderers } const Hardwares::StreamingUploadTicket* const ticket = rrm->FindStreamingUploadTicket(source_radiance); - if (!ticket || !ticket->CompletionTimeline) - return; - - uint64_t completed_upload_value = 0; - vkGetSemaphoreCounterValue(Device->LogicalDevice, ticket->CompletionTimeline->GetHandle(), &completed_upload_value); - if (completed_upload_value < ticket->CompletionValue) - return; + if (ticket) + { + if (!ticket->CompletionTimeline) + return; + uint64_t completed_upload_value = 0; + vkGetSemaphoreCounterValue(Device->LogicalDevice, ticket->CompletionTimeline->GetHandle(), &completed_upload_value); + if (completed_upload_value < ticket->CompletionValue) + return; + } + else + { + // A prior bindless pass may have consumed this ticket before + // sky preparation claimed the source. Its absence alone is + // not readiness: observe the exact graphics acquire completion. + auto retained = rrm->PinTextureVersion(source_radiance); + if (!retained) + return; + const auto acquired = rrm->PollSubmittedTextureAcquire(retained); + if (acquired == SceneTextureDescriptorState::Pending) + return; + if (acquired != SceneTextureDescriptorState::Ready) + { + rrm->ForgetTextureDecode(source_radiance); + m_sky_environment.CompleteBake(revision, source_radiance, false); + DiscardSkyTexture(source_radiance); + ZENGINE_CORE_ERROR("[SkyEnvironment] Revision {} could not acquire its HDRI source", revision) + StartPendingSkyBake(); + return; + } + } EnvironmentLightingResources lighting = CreateSkyLightingResources(bake->BakeSettings); if (!lighting.Valid() || !m_sky_environment.AttachBakeLighting(revision, lighting) || !m_sky_environment.BeginGpuBake(revision)) diff --git a/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.h b/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.h index e0ac6dbea..fd9be1ee0 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.h +++ b/ZEngine/ZEngine/Rendering/Renderers/GraphicRenderer.h @@ -96,6 +96,7 @@ namespace ZEngine::Rendering::Renderers void ApplySelectionOutline(bool enabled); /// @brief Configures an optional one-pixel object-ID readback for the next frame. void RequestObjectIdPick(const ObjectIdPickRequest& request); + void RequestSceneColorCapture(RGReadbackFn complete, RGReadbackCancelFn cancelled, void* context); /// @brief Returns the stable frame-color texture used by UI viewport widgets. Textures::TextureHandle GetFrameOutput(); /// @brief Returns a coherent renderer-memory snapshot for a non-render-thread UI reader. @@ -107,7 +108,7 @@ namespace ZEngine::Rendering::Renderers private: friend struct ::RRMTestHelper; - bool SubmitSkySnapshot(const Scenes::SkyConfig& sky, const Scenes::SkyCelestialLight& celestial_light, uint64_t revision, Scenes::SkyEnvironmentPublication publication); + bool SubmitSkySnapshot(const Scenes::SkyConfig& sky, const Scenes::SkyCelestialLight& celestial_light, uint64_t revision, Scenes::SkyEnvironmentPublication publication, const RenderResourceManager::SceneEnvironmentPreparation* preparation = nullptr); void ReleaseSceneEnvironmentPreparation(); void EndScenePipelinePreparation(); enum class SceneTextureSourceState : uint8_t diff --git a/ZEngine/ZEngine/Rendering/Renderers/Graphics/ToneMappingPass.cpp b/ZEngine/ZEngine/Rendering/Renderers/Graphics/ToneMappingPass.cpp index 630ead825..b59b1989b 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/Graphics/ToneMappingPass.cpp +++ b/ZEngine/ZEngine/Rendering/Renderers/Graphics/ToneMappingPass.cpp @@ -5,6 +5,12 @@ using namespace ZEngine::Rendering::Specifications; namespace ZEngine::Rendering::Renderers { + void ToneMappingPass::CancelPendingCapture() + { + if (m_capture.Context && m_capture.Cancelled) + m_capture.Cancelled(m_capture.Context); + m_capture = {}; + } void ToneMappingPass::SetUseCompositedSceneColor(bool enabled) { m_use_composited_scene_color = enabled; @@ -22,7 +28,19 @@ namespace ZEngine::Rendering::Renderers res_builder->ReadTexture(m_use_composited_scene_color ? RendererResourceName::FrameHdrCompositedRenderTargetName : RendererResourceName::FrameHdrColorRenderTargetName, "SceneColor"); const auto target = DescribeSceneTarget(SceneRenderTarget::Color, width, height); - res_builder->WriteColorAttachment(target.Name, target.Spec); + const auto output = res_builder->WriteColorAttachment(target.Name, target.Spec); + if (m_capture.Context) + { + const RGImageReadbackRegion region = { + .Subresource = {.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, .mipLevel = 0, .baseArrayLayer = 0, .layerCount = 1}, + .Offset = {0, 0, 0}, + .Extent = {width, height, 1} + }; + const auto capture = res_builder->DeclareImageReadback("SceneColorCapture", output, region, m_capture.Complete, m_capture.Context, VkDeviceSize(width) * height * 4, m_capture.Cancelled, true); + if (!capture.Valid() && m_capture.Cancelled) + m_capture.Cancelled(m_capture.Context); + m_capture = {}; + } return true; } diff --git a/ZEngine/ZEngine/Rendering/Renderers/Graphics/ToneMappingPass.h b/ZEngine/ZEngine/Rendering/Renderers/Graphics/ToneMappingPass.h index e980f822f..d463f26c4 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/Graphics/ToneMappingPass.h +++ b/ZEngine/ZEngine/Rendering/Renderers/Graphics/ToneMappingPass.h @@ -7,6 +7,17 @@ namespace ZEngine::Rendering::Renderers /// @brief Converts the graph's linear HDR scene colour into the editor's display texture. struct ToneMappingPass final : public IRenderGraphCallbackPass { + struct CaptureRequest + { + RGReadbackFn Complete = nullptr; + RGReadbackCancelFn Cancelled = nullptr; + void* Context = nullptr; + }; + void RequestCapture(const CaptureRequest& request) + { + m_capture = request; + } + void CancelPendingCapture(); /// @brief Uses atmosphere-composited HDR color when that per-view path is active. void SetUseCompositedSceneColor(bool enabled); bool Register(Hardwares::VulkanDevicePtr const device, cstring name, const RenderGraphFrameContext& frame_context, RenderGraphResourceBuilderPtr const res_builder, RenderGraphResourceInspectorPtr res_inspector) override; @@ -20,6 +31,7 @@ namespace ZEngine::Rendering::Renderers } private: - bool m_use_composited_scene_color = false; + bool m_use_composited_scene_color = false; + CaptureRequest m_capture = {}; }; } // namespace ZEngine::Rendering::Renderers diff --git a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp index a5294de5e..5e4158a2b 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp +++ b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.cpp @@ -409,21 +409,46 @@ namespace ZEngine::Rendering::Renderers if (!graph || !CanRecordSecondary(graph->Device, pass) || pass.SecondaryWorker == UINT32_MAX) return; +#if ZENGINE_PROFILING + auto stage_start = std::chrono::steady_clock::now(); + const auto report_slow_stage = [&stage_start](const char* name) { + const auto now = std::chrono::steady_clock::now(); + const double elapsed_ms = std::chrono::duration(now - stage_start).count(); + if (elapsed_ms >= 1.0) + ZENGINE_PROFILE_VALUE(name, elapsed_ms); + stage_start = now; + }; +#endif auto* secondary = graph->Device->CommandBufferMgr->AcquireWorkerSecondary(pass.Queue, frame_index, pass.SecondaryWorker, pass.SecondaryOrdinal); pass.Secondary = secondary; +#if ZENGINE_PROFILING + report_slow_stage("SceneLoad.RecordingWorker.AcquireMs"); +#endif if (IsGraphicPass(pass)) { secondary->BeginSecondary(static_cast(pass.Handle), ResolveGraphFramebuffer(graph->Device, pass)); +#if ZENGINE_PROFILING + report_slow_stage("SceneLoad.RecordingWorker.BeginMs"); +#endif pass.SecondaryRecorded = pass.Callback->RecordDraw(graph->Device, graph->ResourceInspector, graph->SceneData, pass.Handle, pass.Framebuffer, secondary); } else { secondary->BeginSecondaryCompute(); +#if ZENGINE_PROFILING + report_slow_stage("SceneLoad.RecordingWorker.BeginMs"); +#endif pass.Callback->Execute(graph->Device, graph->ResourceInspector, graph->SceneData, pass.Handle, pass.Framebuffer, secondary); pass.SecondaryRecorded = true; } +#if ZENGINE_PROFILING + report_slow_stage("SceneLoad.RecordingWorker.CallbackMs"); +#endif secondary->End(); +#if ZENGINE_PROFILING + report_slow_stage("SceneLoad.RecordingWorker.EndMs"); +#endif } void RecordGraphSecondaryWorkerBatch(void* context) @@ -3606,7 +3631,7 @@ namespace ZEngine::Rendering::Renderers Device->SwapchainPtr->DirectGraphicsTimeline = timeline; Device->SwapchainPtr->DirectGraphicsTimelineValue = signal_value; } - AcknowledgeStreamingAcquires(batch.FirstPassOrder, batch.PassCount); + AcknowledgeStreamingAcquires(batch.FirstPassOrder, batch.PassCount, timeline, signal_value); SubmitReadbacks(batch.FirstPassOrder, batch.PassCount, timeline, signal_value); // RenderGraph and DeviceSwapchain run on the render thread. Appending // directly avoids re-draining the MPSC producer queue while ensuring @@ -5743,6 +5768,23 @@ namespace ZEngine::Rendering::Renderers // compute or transfer. if (!pass.QueryWrites.empty()) pass.RequestedQueue = Rendering::QueueType::GRAPHIC_QUEUE; + // Streamed images are released to the graphics family. Untyped + // bindless reads cannot express per-image queue ownership, and a + // ticketed explicit import must acquire on that same family. + const auto reads_ticketed_image = [&](const auto& uses) { + for (const auto& use : uses) + { + if (use.Handle.Valid() && use.Handle.Index < Resources.size() && Resources[use.Handle.Index].HasStreamingTicket) + { + return true; + } + } + return false; + }; + if (pass.ReadsBindless || reads_ticketed_image(pass.Reads) || reads_ticketed_image(pass.Writes)) + { + pass.RequestedQueue = Rendering::QueueType::GRAPHIC_QUEUE; + } } bool isolate_atmosphere_view_passes = false; #ifndef NDEBUG @@ -5923,7 +5965,7 @@ namespace ZEngine::Rendering::Renderers plan_ticket(ticket, UINT32_MAX); } - void RenderGraph::AcknowledgeStreamingAcquires(uint32_t first_pass_order, uint32_t pass_count) + void RenderGraph::AcknowledgeStreamingAcquires(uint32_t first_pass_order, uint32_t pass_count, Rendering::Primitives::Semaphore* timeline, uint64_t value) { auto* const rrm = Device && Device->RRM ? static_cast(Device->RRM) : nullptr; const uint32_t end_order = std::min(first_pass_order + pass_count, static_cast(SortedPassIndices.size())); @@ -5935,8 +5977,10 @@ namespace ZEngine::Rendering::Renderers if (!plan.BarrierRecorded || plan.Acknowledged) continue; - if (rrm) - rrm->AcknowledgeStreamingUploadTicket(plan.Ticket); + if (rrm && !rrm->AcknowledgeStreamingUploadAcquire(plan.Ticket, timeline, value, Device->GetQueue(pass.Queue).FamilyIndex)) + { + continue; + } for (RGImportedResource& imported : ImportedResources) { const auto& ticket = imported.StreamingTicket; @@ -5955,7 +5999,7 @@ namespace ZEngine::Rendering::Renderers auto* const graph = static_cast(context); if (graph) { - graph->AcknowledgeStreamingAcquires(0, static_cast(graph->SortedPassIndices.size())); + graph->AcknowledgeStreamingAcquires(0, static_cast(graph->SortedPassIndices.size()), timeline, timeline_value); graph->SubmitReadbacks(0, static_cast(graph->SortedPassIndices.size()), timeline, timeline_value); } } diff --git a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.h b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.h index 79984c122..a68fdaad6 100644 --- a/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.h +++ b/ZEngine/ZEngine/Rendering/Renderers/RenderGraph.h @@ -765,6 +765,7 @@ namespace ZEngine::Rendering::Renderers struct RenderGraph : IScenePipelinePreparation { friend struct RenderGraphTestAccess; + friend struct ::RRMUploadBatchTestHelper; static constexpr uint32_t MaxScenePipelineVariants = 32; @@ -937,7 +938,7 @@ namespace ZEngine::Rendering::Renderers void CancelUnsubmittedReadbacks(bool preserve_recorded = false); void MarkQueryWritesRecorded(const RGPass& pass); void BuildStreamingAcquirePlans(); - void AcknowledgeStreamingAcquires(uint32_t first_pass_order, uint32_t pass_count); + void AcknowledgeStreamingAcquires(uint32_t first_pass_order, uint32_t pass_count, Rendering::Primitives::Semaphore* timeline, uint64_t value); static void OnRenderWorkSubmitted(void* context, Rendering::Primitives::Semaphore* timeline, uint64_t timeline_value); static void OnRenderWorkCancelled(void* context); // Returns false when the enabled subgraph has a dependency cycle. A cycle diff --git a/ZEngine/ZEngine/Rendering/Scenes/CookedSceneResourceRequest.cpp b/ZEngine/ZEngine/Rendering/Scenes/CookedSceneResourceRequest.cpp new file mode 100644 index 000000000..2895d42a5 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Scenes/CookedSceneResourceRequest.cpp @@ -0,0 +1,197 @@ +#include +#include +#include +#include +#include + +namespace ZEngine::Rendering::Scenes +{ + CookedSceneResourceRequest::~CookedSceneResourceRequest() + { + const bool reset = Reset(); + ZENGINE_VALIDATE_ASSERT(reset, "Cooked scene request destroyed before cancellation completed") + (void) reset; + } + + bool CookedSceneResourceRequest::Begin( + ZEngine::ECS::SceneInstance& instance, + RenderScene& render, + SceneResourcePreparation& preparation, + std::span meshes, + ZEngine::Managers::CookedCpuClosureCache::Lease&& assets, + ZEngine::ECS::SceneRequestId request, + uint64_t generation, + const ZEngine::Rendering::Scenes::SceneInitialView& view, + ZEngine::Core::Memory::IAllocationRoute& route, + ZEngine::Core::Memory::TLSFSlab& metadata, + ZEngine::Rendering::RenderResourceManager& resources, + ZEngine::Hardwares::VulkanDevice& device, + ZEngine::Rendering::CookedSceneGpuMailbox& mailbox, + uint32_t max_textures, + bool pause_after_mesh_record) + { + ZEngine::Managers::CookedCpuClosureCache::Result result = {}; + if (m_state != State::Empty || !request || !generation || generation == std::numeric_limits::max() || device.RRM != &resources || !assets.Poll(&result) || result.StateValue != ZEngine::Managers::CookedCpuClosureCache::State::Ready || !result.Assets || !instance.MatchesCookedClosure(result.Assets->ManifestDigest(), result.Assets->SceneUUID()) || preparation.HasPreparation() || max_textures > ZEngine::Rendering::RenderResourceManager::SceneCookedTextureCapacity) + { + return false; + } + m_instance = &instance; + m_render = &render; + m_preparation = &preparation; + m_meshes = meshes; + m_assets = std::move(assets); + m_request = request; + m_generation = generation; + m_view = view; + m_route = &route; + m_metadata = &metadata; + m_resources = &resources; + m_device = &device; + m_mailbox = &mailbox; + m_max_textures = max_textures; + m_pause_after_mesh_record = pause_after_mesh_record; + m_mesh_upload_recorded = false; + m_state = State::Collecting; + return true; + } + + void CookedSceneResourceRequest::Finish(State outcome) noexcept + { + m_state = outcome; + } + + void CookedSceneResourceRequest::ReleaseTerminalAssets() noexcept + { + if (m_state == State::Failed || m_state == State::Cancelled) + m_assets.Reset(); + } + + void CookedSceneResourceRequest::Cancel() noexcept + { + if (m_state == State::Empty || m_state == State::Failed || m_state == State::Cancelled || m_state == State::Cancelling) + return; + if (m_ticket) + { + (void) m_mailbox->Cancel(m_ticket); + m_after_cancel = State::Cancelled; + m_state = State::Cancelling; + return; + } + Finish(State::Cancelled); + } + + CookedSceneResourceRequest::State CookedSceneResourceRequest::Tick() + { + using Mailbox = ZEngine::Rendering::CookedSceneGpuMailbox; + using Resource = ZEngine::Rendering::RenderResourceManager; + if (m_state == State::GpuPending || m_state == State::Cancelling) + { + m_mesh_upload_recorded |= m_mailbox->MeshUploadRecorded(m_ticket); + Mailbox::Outcome outcome = {}; + if (!m_mailbox->Poll(m_ticket, &outcome)) + { + // Ticket loss violates the owner contract; do not discard a + // candidate that might still be borrowed by the render owner. + return m_state; + } + if (outcome == Mailbox::Outcome::Pending) + return m_state; + if (!m_mailbox->Acknowledge(m_ticket)) + return m_state; + m_ticket = {}; + if (m_state == State::Cancelling) + { + Finish(m_after_cancel); + } + else if (outcome == Mailbox::Outcome::Submitted) + { + m_state = State::ResourcePending; + } + else + { + Finish(State::Failed); + } + } + if (m_state == State::Collecting) + { + ZEngine::Managers::CookedCpuClosureCache::Result result = {}; + if (m_resources->SceneResourceAdmissionSealed() || !m_assets.Poll(&result) || result.StateValue != ZEngine::Managers::CookedCpuClosureCache::State::Ready || !result.Assets) + { + Finish(State::Failed); + return m_state; + } + bool terminal_failure = false; + if (!m_preparation->Prepare(*m_instance, *m_render, m_meshes, *m_resources, m_request, m_generation, m_view, result.Assets, &terminal_failure)) + { + if (terminal_failure) + Finish(State::Failed); + return m_state; + } + Resource::SceneResourcePreparationResult collecting = {}; + if (!m_preparation->Poll(&collecting) || collecting.State == Resource::SceneResourcePreparationState::Failed || collecting.State == Resource::SceneResourcePreparationState::Releasing) + { + Finish(State::Failed); + return m_state; + } + if (!m_preparation->InstancesCollected(m_render->Instances.size())) + return m_state; + const auto& sky = m_render->Sky; + const ZEngine::Cook::CookAssetIdentity environment = sky.IsHDRI() ? ZEngine::Cook::CookAssetIdentity{ZEngine::Cook::CookAssetKind::EnvironmentMap, sky.EnvironmentMap} : ZEngine::Cook::CookAssetIdentity{}; + const auto admission = m_mailbox->TryBegin(m_assets, m_meshes, environment, *m_route, *m_metadata, *m_resources, *m_device, m_preparation->InitialResources(), m_max_textures, &m_ticket, m_pause_after_mesh_record); + if (admission == Mailbox::Admission::Failed) + { + Finish(State::Failed); + } + else if (admission == Mailbox::Admission::Queued) + { + m_state = State::GpuPending; + } + } + if (m_state == State::ResourcePending) + { + Resource::SceneResourcePreparationResult result = {}; + if (!m_preparation->Poll(&result) || result.State == Resource::SceneResourcePreparationState::Failed || result.State == Resource::SceneResourcePreparationState::Releasing) + { + ZENGINE_CORE_ERROR("[CookedSceneResourceRequest] Resource preparation failed: state={} error={} material_error={} environment={}", static_cast(result.State), static_cast(result.Error), static_cast(result.MaterialError), static_cast(result.Environment)) + Finish(State::Failed); + } + else if (result.State == Resource::SceneResourcePreparationState::ResourcesRetained) + { + m_state = State::Ready; + } + } + return m_state; + } + + bool CookedSceneResourceRequest::TakeReady() noexcept + { + if (m_state != State::Ready) + return false; + m_assets.Reset(); + m_state = State::Empty; + return Reset(); + } + + bool CookedSceneResourceRequest::Reset() noexcept + { + if (m_ticket || (m_state != State::Empty && m_state != State::Failed && m_state != State::Cancelled)) + return false; + m_assets.Reset(); + m_instance = nullptr; + m_render = nullptr; + m_preparation = nullptr; + m_meshes = {}; + m_route = nullptr; + m_metadata = nullptr; + m_resources = nullptr; + m_device = nullptr; + m_mailbox = nullptr; + m_request = {}; + m_generation = 0; + m_max_textures = 0; + m_pause_after_mesh_record = false; + m_mesh_upload_recorded = false; + m_state = State::Empty; + return true; + } +} // namespace ZEngine::Rendering::Scenes diff --git a/ZEngine/ZEngine/Rendering/Scenes/CookedSceneResourceRequest.h b/ZEngine/ZEngine/Rendering/Scenes/CookedSceneResourceRequest.h new file mode 100644 index 000000000..b211a20e3 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Scenes/CookedSceneResourceRequest.h @@ -0,0 +1,73 @@ +#pragma once +#include +#include +#include + +namespace ZEngine::ECS +{ + class SceneInstance; +} + +namespace ZEngine::Rendering::Scenes +{ + /// Owner-thread transaction for an unpublished cooked scene. The caller + /// keeps the scene, render adapter, mesh span and services alive until a + /// terminal result or TakeReady; cancellation must drain its mailbox ticket. + class CookedSceneResourceRequest final + { + public: + enum class State : uint8_t + { + Empty, + Collecting, + GpuPending, + ResourcePending, + Cancelling, + Ready, + Failed, + Cancelled + }; + CookedSceneResourceRequest() = default; + ~CookedSceneResourceRequest(); + CookedSceneResourceRequest(const CookedSceneResourceRequest&) = delete; + CookedSceneResourceRequest& operator=(const CookedSceneResourceRequest&) = delete; + + [[nodiscard]] bool Begin(ECS::SceneInstance&, RenderScene&, SceneResourcePreparation&, std::span, Managers::CookedCpuClosureCache::Lease&&, ECS::SceneRequestId, uint64_t, const SceneInitialView&, Core::Memory::IAllocationRoute&, Core::Memory::TLSFSlab&, RenderResourceManager&, Hardwares::VulkanDevice&, CookedSceneGpuMailbox&, uint32_t, bool pause_after_mesh_record = false); + [[nodiscard]] State Tick(); + void Cancel() noexcept; + [[nodiscard]] bool TakeReady() noexcept; + /// Call after retiring a failed/cancelled candidate; preserves status. + void ReleaseTerminalAssets() noexcept; + [[nodiscard]] bool Reset() noexcept; + [[nodiscard]] State Status() const noexcept + { + return m_state; + } + [[nodiscard]] bool MeshUploadRecorded() const noexcept + { + return m_mesh_upload_recorded; + } + + private: + void Finish(State) noexcept; + ECS::SceneInstance* m_instance = nullptr; + RenderScene* m_render = nullptr; + SceneResourcePreparation* m_preparation = nullptr; + std::span m_meshes = {}; + Managers::CookedCpuClosureCache::Lease m_assets = {}; + Core::Memory::IAllocationRoute* m_route = nullptr; + Core::Memory::TLSFSlab* m_metadata = nullptr; + RenderResourceManager* m_resources = nullptr; + Hardwares::VulkanDevice* m_device = nullptr; + CookedSceneGpuMailbox* m_mailbox = nullptr; + CookedSceneGpuMailbox::Ticket m_ticket = {}; + ECS::SceneRequestId m_request = {}; + SceneInitialView m_view = {}; + uint64_t m_generation = 0; + uint32_t m_max_textures = 0; + bool m_pause_after_mesh_record = false; + bool m_mesh_upload_recorded = false; + State m_after_cancel = State::Cancelled; + State m_state = State::Empty; + }; +} // namespace ZEngine::Rendering::Scenes diff --git a/ZEngine/ZEngine/Rendering/Scenes/RuntimeScene.cpp b/ZEngine/ZEngine/Rendering/Scenes/RuntimeScene.cpp new file mode 100644 index 000000000..6a36f0d5d --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Scenes/RuntimeScene.cpp @@ -0,0 +1,68 @@ +#include +#include +#include + +namespace ZEngine::Rendering::Scenes +{ + bool RuntimeScene::PrepareFromCooked(std::unique_ptr&& instance, const Managers::CookedCpuAssetSet& assets, std::unique_ptr* output, ECS::SceneDiagnostics* diagnostics, const SceneMeshBindingLimits& limits) + { + if (!output || *output || !instance || !assets.IsValid() || !instance->MatchesCookedClosure(assets.ManifestDigest(), assets.SceneUUID()) || !instance->Document().Settings() || !instance->Arena() || instance->Actors().Count() || !instance->Commands().IsEmpty() || instance->Tick().SystemCount()) + { + ECS::SceneDiagnosticError(diagnostics, "Runtime binding requires an unpublished scene and its verified cooked closure."); + return false; + } + auto candidate = std::unique_ptr(new (std::nothrow) RuntimeScene{}); + if (!candidate) + return false; + candidate->m_instance = std::move(instance); + candidate->m_render.Instances.m_allocator = candidate->m_instance->Arena(); + const auto* closure = &assets; + const SceneMeshResolver resolver = {&closure, [](void* context, const uuids::uuid& uuid, uint32_t* draws, ECS::SceneDiagnostics*) { + const auto& set = **static_cast(context); + const auto* mesh = set.FindMesh({Cook::CookAssetKind::Mesh, uuid}); + if (!draws || !mesh || mesh->SubMeshes.empty() || mesh->SubMeshes.size() > UINT32_MAX) + return false; + *draws = static_cast(mesh->SubMeshes.size()); + return true; + }}; + if (!BuildSceneMeshBindings(*candidate->m_instance, candidate->m_render, resolver, limits, candidate->m_meshes, candidate->m_draw_count, diagnostics)) + return false; + const auto& settings = *candidate->m_instance->Document().Settings(); + candidate->m_render.Sky = settings.Sky; + if (!MakeSceneGridRenderSnapshot(settings.Grid, candidate->m_render.Grid)) + { + candidate->m_render.Grid = {}; + candidate->m_render.Grid.Enabled = false; + } + candidate->m_render.MarkInstancesDirty(); + candidate->m_render.MarkSkyDirty(); + candidate->m_instance->RecordMemoryUsage(); + *output = std::move(candidate); + return true; + } + ECS::SceneInstance& RuntimeScene::Instance() noexcept + { + return *m_instance; + } + + RenderScene& RuntimeScene::Render() noexcept + { + return m_render; + } + + SceneResourcePreparation& RuntimeScene::Resources() noexcept + { + return m_resources; + } + + std::span RuntimeScene::RequiredMeshes() const noexcept + { + return {m_meshes.data(), m_meshes.size()}; + } + + uint32_t RuntimeScene::DrawCount() const noexcept + { + return m_draw_count; + } + +} // namespace ZEngine::Rendering::Scenes diff --git a/ZEngine/ZEngine/Rendering/Scenes/RuntimeScene.h b/ZEngine/ZEngine/Rendering/Scenes/RuntimeScene.h new file mode 100644 index 000000000..f46cb5b6c --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Scenes/RuntimeScene.h @@ -0,0 +1,46 @@ +#pragma once +#include +#include +#include + +namespace ZEngine::Managers +{ + class CookedCpuAssetSet; +} + +namespace ZEngine::Rendering::Scenes +{ + /// @brief Owns an unpublished or active renderer-only scene and its bindings. + /// @details Scene storage outlives the render adapter and retained resources. + /// Retire this owner before the resource manager and cache dependencies shut down. + class RuntimeScene final + { + public: + ~RuntimeScene() = default; + RuntimeScene(const RuntimeScene&) = delete; + RuntimeScene& operator=(const RuntimeScene&) = delete; + + /// @brief Build render bindings from a verified cooked asset closure. + /// @details Success consumes the instance. Validation failure preserves it; + /// failures after binding starts consume and discard it. + [[nodiscard]] static bool PrepareFromCooked(std::unique_ptr&&, const Managers::CookedCpuAssetSet&, std::unique_ptr*, ECS::SceneDiagnostics*, const SceneMeshBindingLimits& = {}); + /// @brief Return the owned scene instance. + [[nodiscard]] ECS::SceneInstance& Instance() noexcept; + /// @brief Return the scene's render adapter. + [[nodiscard]] RenderScene& Render() noexcept; + /// @brief Return the owner of the scene's prepared render resources. + [[nodiscard]] SceneResourcePreparation& Resources() noexcept; + /// @brief Return the mesh identities needed by the render bindings. + [[nodiscard]] std::span RequiredMeshes() const noexcept; + /// @brief Return the number of prepared draw instances. + [[nodiscard]] uint32_t DrawCount() const noexcept; + + private: + RuntimeScene() = default; + std::unique_ptr m_instance = {}; + RenderScene m_render = {}; + Core::Containers::Array m_meshes = {}; + uint32_t m_draw_count = 0; + SceneResourcePreparation m_resources = {}; + }; +} // namespace ZEngine::Rendering::Scenes diff --git a/ZEngine/ZEngine/Rendering/Scenes/SceneMeshBindings.cpp b/ZEngine/ZEngine/Rendering/Scenes/SceneMeshBindings.cpp new file mode 100644 index 000000000..a08ab476c --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Scenes/SceneMeshBindings.cpp @@ -0,0 +1,111 @@ +#include +#include +#include +#include +#include +#include + +using namespace ZEngine::ECS; +using namespace ZEngine::ECS::Components; + +namespace ZEngine::Rendering::Scenes +{ + namespace + { + bool Reject(SceneDiagnostics* diagnostics, cstring message) + { + SceneDiagnosticError(diagnostics, message); + return false; + } + struct MeshRequirement + { + uuids::uuid UUID = {}; + uint32_t Uses = 1; + }; + } // namespace + + bool BuildSceneMeshBindings(SceneInstance& instance, RenderScene& render, const SceneMeshResolver& assets, const SceneMeshBindingLimits& limits, Core::Containers::Array& required_meshes, uint32_t& draw_count, SceneDiagnostics* diagnostics) + { + if (!instance.Arena() || !render.Instances.m_allocator || !render.Instances.empty() || !required_meshes.empty() || draw_count || limits.MaxInstances > SceneData::MAX_DRAW_COMMANDS || limits.MaxDrawCommands > SceneData::MAX_DRAW_COMMANDS) + return Reject(diagnostics, "Invalid unpublished scene mesh binding inputs."); + auto& world = instance.World(); + // O(E + M log M), with one asset lookup per distinct mesh, not per placed + // instance. Scratch is charged to the bounded candidate scene arena. + ZEngine::Core::Containers::Array requirements = {}; + requirements.m_allocator = instance.Arena(); + bool valid = true; + world.ForEach([&](EntityID entity, const MeshComponent& mesh) { + if (!world.HasComponent(entity) || mesh.MeshUUID.is_nil() || requirements.size() >= limits.MaxInstances) + { + valid = false; + return; + } + if (!requirements.try_push({mesh.MeshUUID})) + valid = false; + }); + if (!valid) + { + return Reject(diagnostics, "A mesh has no transform/asset identity, or the complete scene exceeds the instance budget."); + } + const auto instance_count = static_cast(requirements.size()); + if (instance_count && !assets.ResolveMesh) + { + return Reject(diagnostics, "Scene meshes need an explicit asset resolver; no render bindings were published."); + } + if (!requirements.empty()) + std::sort(requirements.begin(), requirements.end(), [](const auto& a, const auto& b) { return a.UUID < b.UUID; }); + size_t unique_count = 0; + for (size_t i = 0; i < requirements.size(); ++i) + { + if (unique_count && requirements[unique_count - 1].UUID == requirements[i].UUID) + { + ++requirements[unique_count - 1].Uses; + } + else + { + requirements[unique_count++] = requirements[i]; + } + } + requirements.m_size = unique_count; + required_meshes.m_allocator = instance.Arena(); + if (!required_meshes.try_reserve(unique_count)) + { + return Reject(diagnostics, "Cannot retain the complete scene resource identity set."); + } + for (const auto& requirement : requirements) + { + required_meshes.push(requirement.UUID); + uint32_t draws = 0; + if (!assets.ResolveMesh(assets.Context, requirement.UUID, &draws, diagnostics) || draws == 0) + { + return Reject(diagnostics, "A referenced mesh is missing or has no complete drawable binding; the active scene was preserved."); + } + const uint64_t additional = uint64_t(draws) * requirement.Uses; + if (additional > limits.MaxDrawCommands - draw_count) + { + return Reject(diagnostics, "The complete scene exceeds the GPU submesh draw budget; no instances were truncated."); + } + draw_count += static_cast(additional); + } + if (!render.Instances.try_reserve(instance_count)) + { + return Reject(diagnostics, "Cannot retain the complete scene instance list in its bounded arena."); + } + world.ForEach([&](EntityID entity, MeshComponent& mesh) { + ZEngine::Rendering::Scenes::MeshInstance instance = {}; + instance.Id = render.NextInstanceId++; + instance.MeshUUID = mesh.MeshUUID; + instance.Entity = entity; + instance.Transform = world.GetComponent(entity)->WorldTransform; + if (const auto* name = world.GetComponent(entity)) + { + ZEngine::Helpers::secure_strncpy(instance.Name, sizeof(instance.Name), name->Value, sizeof(instance.Name) - 1); + } + mesh.RenderInstanceId = instance.Id; + render.Instances.push(instance); + }); + + instance.RecordMemoryUsage(); + return true; + } +} // namespace ZEngine::Rendering::Scenes diff --git a/ZEngine/ZEngine/Rendering/Scenes/SceneMeshBindings.h b/ZEngine/ZEngine/Rendering/Scenes/SceneMeshBindings.h new file mode 100644 index 000000000..6edbb093b --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Scenes/SceneMeshBindings.h @@ -0,0 +1,24 @@ +#pragma once +#include +#include + +namespace ZEngine::Rendering::Scenes +{ + /// Resolves CPU mesh draw counts on the exclusive scene owner. It must not + /// mutate active bindings; returned counts cover all submeshes of a mesh. + struct SceneMeshResolver + { + void* Context = nullptr; + bool (*ResolveMesh)(void*, const uuids::uuid&, uint32_t*, ECS::SceneDiagnostics*) = nullptr; + }; + struct SceneMeshBindingLimits + { + uint32_t MaxInstances = SceneData::MAX_DRAW_COMMANDS; + uint32_t MaxDrawCommands = SceneData::MAX_DRAW_COMMANDS; + }; + + /// Builds an unpublished scene's complete mesh identities and instances in + /// its bounded arena. Render instances must be initialized and empty. On + /// failure the caller discards the candidate, including any partial output. + [[nodiscard]] bool BuildSceneMeshBindings(ECS::SceneInstance&, RenderScene&, const SceneMeshResolver&, const SceneMeshBindingLimits&, Core::Containers::Array&, uint32_t&, ECS::SceneDiagnostics*); +} // namespace ZEngine::Rendering::Scenes diff --git a/ZEngine/ZEngine/Rendering/Scenes/SceneResourcePreparation.cpp b/ZEngine/ZEngine/Rendering/Scenes/SceneResourcePreparation.cpp new file mode 100644 index 000000000..792d6ed55 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Scenes/SceneResourcePreparation.cpp @@ -0,0 +1,99 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ZEngine::Rendering::Scenes +{ + SceneResourcePreparation::~SceneResourcePreparation() + { + if (m_resources && m_preparation) + m_resources->ReleaseSceneResourcePreparation(m_preparation); + } + + bool SceneResourcePreparation::Prepare(ECS::SceneInstance& instance, RenderScene& render, std::span meshes, RenderResourceManager& resources, ECS::SceneRequestId request, uint64_t generation, const SceneInitialView& view, const Managers::CookedCpuAssetSet* cooked, bool* terminal_failure) + { + ZENGINE_PROFILE_SCOPE("SceneLoad.Main.ResourceAdmission"); + if (terminal_failure) + *terminal_failure = false; + if (cooked && (!cooked->IsValid() || !instance.MatchesCookedClosure(cooked->ManifestDigest(), cooked->SceneUUID()))) + { + if (terminal_failure) + *terminal_failure = true; + return false; + } + if (m_preparation) + { + if (m_resources != &resources || m_cooked != (cooked != nullptr)) + { + if (terminal_failure) + *terminal_failure = true; + return false; + } + if (m_instances_copied < render.Instances.size()) + { + const auto count = std::min(size_t(64), render.Instances.size() - m_instances_copied); + if (resources.AppendSceneResourceInstances(m_preparation, {render.Instances.data() + m_instances_copied, count})) + m_instances_copied += static_cast(count); + } + return true; + } + auto& world = instance.World(); + ECS::Systems::SyncHierarchy(world); + for (auto& item : render.Instances) + { + const auto* transform = world.GetComponent(item.Entity); + if (!transform) + { + if (terminal_failure) + *terminal_failure = true; + return false; + } + item.Transform = transform->WorldTransform; + } + render.MarkInstancesDirty(); + ECS::Systems::SyncECSToLights(world, render); + const RenderResourceManager::SceneEnvironmentInput environment = {render.Sky, render.CelestialLight, render.SkyRevision.value.load(std::memory_order_acquire)}; + const RenderResourceManager::SceneInitialFrameInput frame = { + {render.Instances.data(), render.Instances.size()}, + render.PendingLights, render.Grid, view + }; + if (!resources.PrepareSceneResources(request, generation, meshes, &m_preparation, RenderResourceManager::ScenePreparationContent::MeshMaterials, &environment, &frame, true, cooked)) + return false; + m_instances_copied = 0; + m_resources = &resources; + m_cooked = cooked != nullptr; + return true; + } + + bool SceneResourcePreparation::Poll(RenderResourceManager::SceneResourcePreparationResult* output) const + { + return m_resources && m_resources->PollSceneResourcePreparation(m_preparation, output); + } + + RenderResourceManager::SceneResourcePreparationId SceneResourcePreparation::InitialResources() + { + RenderResourceManager::SceneResourcePreparationResult result = {}; + if (!m_cooked && Poll(&result) && result.FrameRecorded) + { + m_resources->ReleaseSceneResourcePreparation(m_preparation); + m_preparation = {}; + m_resources = nullptr; + } + return m_preparation; + } + bool SceneResourcePreparation::InstancesCollected(size_t count) const noexcept + { + return bool(m_preparation) && m_instances_copied == count; + } + + bool SceneResourcePreparation::HasPreparation() const noexcept + { + return bool(m_preparation); + } + +} // namespace ZEngine::Rendering::Scenes diff --git a/ZEngine/ZEngine/Rendering/Scenes/SceneResourcePreparation.h b/ZEngine/ZEngine/Rendering/Scenes/SceneResourcePreparation.h new file mode 100644 index 000000000..7c91a9484 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Scenes/SceneResourcePreparation.h @@ -0,0 +1,39 @@ +#pragma once +#include + +namespace ZEngine::ECS +{ + class SceneInstance; +} +namespace ZEngine::Managers +{ + class CookedCpuAssetSet; +} + +namespace ZEngine::Rendering::Scenes +{ + /// Owns one scene's retained GPU-resource preparation. The scene instance, + /// render adapter and resource manager must outlive this owner. + class SceneResourcePreparation final + { + public: + ~SceneResourcePreparation(); + SceneResourcePreparation() = default; + SceneResourcePreparation(const SceneResourcePreparation&) = delete; + SceneResourcePreparation& operator=(const SceneResourcePreparation&) = delete; + + [[nodiscard]] bool Prepare(ECS::SceneInstance&, RenderScene&, std::span, RenderResourceManager&, ECS::SceneRequestId, uint64_t, const SceneInitialView&, const Managers::CookedCpuAssetSet*, bool* terminal_failure); + [[nodiscard]] bool Poll(RenderResourceManager::SceneResourcePreparationResult*) const; + [[nodiscard]] RenderResourceManager::SceneResourcePreparationId InitialResources(); + /// @brief Report whether all expected instances have been collected. + [[nodiscard]] bool InstancesCollected(size_t count) const noexcept; + /// @brief Report whether a resource preparation is active. + [[nodiscard]] bool HasPreparation() const noexcept; + + private: + RenderResourceManager* m_resources = nullptr; + RenderResourceManager::SceneResourcePreparationId m_preparation = {}; + uint32_t m_instances_copied = 0; + bool m_cooked = false; + }; +} // namespace ZEngine::Rendering::Scenes diff --git a/ZEngine/ZEngine/Rendering/Scenes/SceneSubmissionProgress.h b/ZEngine/ZEngine/Rendering/Scenes/SceneSubmissionProgress.h index 5c123089f..caa5f82b2 100644 --- a/ZEngine/ZEngine/Rendering/Scenes/SceneSubmissionProgress.h +++ b/ZEngine/ZEngine/Rendering/Scenes/SceneSubmissionProgress.h @@ -12,15 +12,16 @@ namespace ZEngine::Rendering::Scenes public: struct Snapshot { - uint64_t Generation = 0; - uint64_t Count = 0; - uint64_t FirstTimestampNs = 0; - uint32_t DrawCount = 0; + uint64_t Generation = 0; + uint64_t Count = 0; + uint64_t FirstTimestampNs = 0; + uint32_t DrawCount = 0; + uint64_t FirstTimelineValue = 0; }; /// Called only after Vulkan accepted a complete scene frame. A rejected /// graph, aborted acquire or failed submission must not advance this. - void Record(uint64_t generation, uint64_t timestamp_ns, uint32_t draw_count = 0) noexcept + void Record(uint64_t generation, uint64_t timestamp_ns, uint32_t draw_count = 0, uint64_t timeline_value = 0) noexcept { const auto previous = m_generation.load(); if (!generation || generation < previous) @@ -36,6 +37,7 @@ namespace ZEngine::Rendering::Scenes m_generation.store(generation); m_count.store(1); m_first_timestamp.store(timestamp_ns); + m_first_timeline_value.store(timeline_value); } else if (m_count.load() != std::numeric_limits::max()) { @@ -58,7 +60,7 @@ namespace ZEngine::Rendering::Scenes { continue; } - const Snapshot snapshot = {m_generation.load(), m_count.load(), m_first_timestamp.load(), m_draw_count.load()}; + const Snapshot snapshot = {m_generation.load(), m_count.load(), m_first_timestamp.load(), m_draw_count.load(), m_first_timeline_value.load()}; if (before == m_sequence.load()) { *output = snapshot; @@ -69,10 +71,11 @@ namespace ZEngine::Rendering::Scenes } private: - std::atomic m_sequence = 0; - std::atomic m_generation = 0; - std::atomic m_count = 0; - std::atomic m_first_timestamp = 0; - std::atomic m_draw_count = 0; + std::atomic m_sequence = 0; + std::atomic m_generation = 0; + std::atomic m_count = 0; + std::atomic m_first_timestamp = 0; + std::atomic m_draw_count = 0; + std::atomic m_first_timeline_value = 0; }; } // namespace ZEngine::Rendering::Scenes diff --git a/ZEngine/ZEngine/Rendering/Specifications/TextureSpecification.h b/ZEngine/ZEngine/Rendering/Specifications/TextureSpecification.h index 137ddab2f..1cd4f1262 100644 --- a/ZEngine/ZEngine/Rendering/Specifications/TextureSpecification.h +++ b/ZEngine/ZEngine/Rendering/Specifications/TextureSpecification.h @@ -17,6 +17,9 @@ namespace ZEngine::Rendering::Specifications bool Is3D = false; /// @brief Resizes this transient image when its owning render view changes extent. bool IsRenderTargetSized = false; + /// @brief Charge cooked source assets to the texture budget even when + /// the image supports attachment usage. + bool IsCookedAsset = false; uint32_t Width = 0; uint32_t Height = 0; uint32_t Depth = 1; @@ -54,6 +57,7 @@ namespace ZEngine::Rendering::Specifications ImageCreateFlag ImageCreateFlagValue = ImageCreateFlag::NONE; /// @brief Adds VK_IMAGE_CREATE_ALIAS_BIT when constructing the Vulkan image. bool IsAliasable = false; + bool IsCookedAsset = false; }; } // namespace ZEngine::Rendering::Specifications diff --git a/ZEngine/ZEngine/Rendering/Textures/CookedEnvironmentUpload.cpp b/ZEngine/ZEngine/Rendering/Textures/CookedEnvironmentUpload.cpp new file mode 100644 index 000000000..6fd6df0c5 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Textures/CookedEnvironmentUpload.cpp @@ -0,0 +1,103 @@ +#include +#include + +namespace ZEngine::Rendering::Textures +{ + bool CookedEnvironmentUpload::Capture(Managers::CookedCpuClosureCache::Lease&& lease, const Cook::CookAssetIdentity& identity) + { + Managers::CookedCpuClosureCache::Result result = {}; + if (m_phase != Phase::Empty || identity.Kind != Cook::CookAssetKind::EnvironmentMap || !identity.IsValid() || !lease.Poll(&result) || result.StateValue != Managers::CookedCpuClosureCache::State::Ready || !result.Assets) + { + return false; + } + const auto* version = result.Assets->FindVersion(identity); + const auto* header = result.Assets->FindEnvironment(identity); + const auto object = result.Assets->EnvironmentObject(identity); + if (!version || !header || !Importers::AssetCodec::IsEnvironmentMapFileHeaderValid(*header) || header->HeaderByteSize > object.size() || header->BufferByteSize != object.size() - header->HeaderByteSize) + { + return false; + } + m_header = header; + m_pixels = object.subspan(header->HeaderByteSize); + m_identity = identity; + m_digest = version->ObjectDigest; + m_lease = std::move(lease); + m_phase = Phase::SourceReady; + return true; + } + + bool CookedEnvironmentUpload::Prepare(RenderResourceManager& manager, VkResult* status) + { + if (m_phase != Phase::SourceReady) + { + if (status) + { + *status = VK_ERROR_INITIALIZATION_FAILED; + } + return false; + } + if (!manager.PrepareUploadStaging(m_pixels.size(), &m_staging, status)) + { + return false; + } + m_manager = &manager; + m_phase = Phase::Staging; + return true; + } + + bool CookedEnvironmentUpload::StageOne(VkResult* status) + { + if (m_phase != Phase::Staging || m_staging.Written() >= m_pixels.size()) + { + if (status) + { + *status = VK_ERROR_INITIALIZATION_FAILED; + } + return false; + } + size_t consumed = 0; + if (!m_staging.Append(m_pixels.subspan(static_cast(m_staging.Written())), &consumed, status)) + { + return false; + } + if (m_staging.Ready()) + { + m_phase = Phase::Staged; + } + return true; + } + + bool CookedEnvironmentUpload::Submit(uint8_t frame_index, uint8_t thread_index, TextureHandle target) + { + if (m_phase != Phase::Staged || !m_manager || !m_header || !target.Valid()) + { + return false; + } + const auto* texture = m_manager->GetTexture(target); + if (!texture || texture->Width != m_header->FaceWidth || texture->Height != m_header->FaceHeight || texture->BytePerPixel != sizeof(float) * 4 || texture->Specification.Format != Specifications::ImageFormat::R32G32B32A32_SFLOAT || texture->Specification.Depth != 1 || texture->Specification.LayerCount != m_header->LayerCount || texture->Specification.MipLevelCount != m_header->MipCount || texture->Specification.Is3D || !texture->Specification.IsCubemap || texture->Specification.IsAliasable || !texture->Specification.IsUsageSampled || + !texture->Specification.IsUsageTransfert || !texture->Specification.IsUsageStorage) + { + return false; + } + if (!m_manager->UploadPreparedTextureBuffer(frame_index, thread_index, target, m_staging).Valid()) + { + return false; + } + m_target = target; + m_phase = Phase::Submitted; + return true; + } + + void CookedEnvironmentUpload::Reset() noexcept + { + m_staging.Reset(); + m_lease.Reset(); + m_header = nullptr; + m_pixels = {}; + m_manager = nullptr; + m_identity = {}; + m_digest = {}; + m_target = {}; + m_phase = Phase::Empty; + } +} // namespace ZEngine::Rendering::Textures diff --git a/ZEngine/ZEngine/Rendering/Textures/CookedEnvironmentUpload.h b/ZEngine/ZEngine/Rendering/Textures/CookedEnvironmentUpload.h new file mode 100644 index 000000000..37549d7db --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Textures/CookedEnvironmentUpload.h @@ -0,0 +1,72 @@ +#pragma once +#include +#include + +namespace ZEngine::Rendering::Textures +{ + /// Exclusive staging transaction for the verified base level of a cooked + /// HDR cubemap. Cache, manager and device must outlive this object. The + /// caller owns the destination and must retain it through upload and bake. + class CookedEnvironmentUpload final + { + public: + enum class Phase : uint8_t + { + Empty, + SourceReady, + Staging, + Staged, + Submitted + }; + CookedEnvironmentUpload() = default; + CookedEnvironmentUpload(const CookedEnvironmentUpload&) = delete; + CookedEnvironmentUpload& operator=(const CookedEnvironmentUpload&) = delete; + + [[nodiscard]] bool Capture(Managers::CookedCpuClosureCache::Lease&&, const Cook::CookAssetIdentity&); + [[nodiscard]] bool Prepare(RenderResourceManager&, VkResult* status = nullptr); + [[nodiscard]] bool StageOne(VkResult* status = nullptr); + /// Submission is not proof of GPU completion or consumer acquisition. + [[nodiscard]] bool Submit(uint8_t frame_index, uint8_t thread_index, TextureHandle target); + void Reset() noexcept; + + [[nodiscard]] Phase State() const noexcept + { + return m_phase; + } + [[nodiscard]] const Cook::CookAssetIdentity& Identity() const noexcept + { + return m_identity; + } + [[nodiscard]] const Cook::Digest256& Digest() const noexcept + { + return m_digest; + } + [[nodiscard]] const Importers::AssetCodec::EnvironmentMapFileHeader* Header() const noexcept + { + return m_header; + } + [[nodiscard]] VkDeviceSize StagedBytes() const noexcept + { + return m_staging.Written(); + } + [[nodiscard]] TextureHandle SubmittedTexture() const noexcept + { + return m_phase == Phase::Submitted ? m_target : TextureHandle{}; + } + [[nodiscard]] bool BelongsTo(const RenderResourceManager& manager) const noexcept + { + return m_manager == &manager; + } + + private: + Managers::CookedCpuClosureCache::Lease m_lease = {}; + const Importers::AssetCodec::EnvironmentMapFileHeader* m_header = nullptr; + std::span m_pixels = {}; + RenderResourceManager* m_manager = nullptr; + RenderResourceManager::UploadStaging m_staging = {}; + Cook::CookAssetIdentity m_identity = {}; + Cook::Digest256 m_digest = {}; + TextureHandle m_target = {}; + Phase m_phase = Phase::Empty; + }; +} // namespace ZEngine::Rendering::Textures diff --git a/ZEngine/ZEngine/Rendering/Textures/CookedSceneEnvironmentBatch.cpp b/ZEngine/ZEngine/Rendering/Textures/CookedSceneEnvironmentBatch.cpp new file mode 100644 index 000000000..57836e9f6 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Textures/CookedSceneEnvironmentBatch.cpp @@ -0,0 +1,119 @@ +#include +#include +#include + +namespace ZEngine::Rendering::Textures +{ + CookedSceneEnvironmentBatch::~CookedSceneEnvironmentBatch() + { + Reset(); + } + + bool CookedSceneEnvironmentBatch::Initialize(Managers::CookedCpuClosureCache::Lease&& lease, const Cook::CookAssetIdentity& identity, RenderResourceManager& manager, Hardwares::VulkanDevice& device, RenderResourceManager::SceneResourcePreparationId scene, bool transfer_source) + { + Managers::CookedCpuClosureCache::Result result = {}; + RenderResourceManager::SceneResourcePreparationResult preparation = {}; + RenderResourceManager::SceneEnvironmentPreparation environment = {}; + if (m_state != State::Empty || !scene || device.RRM != &manager || identity.Kind != Cook::CookAssetKind::EnvironmentMap || !identity.IsValid() || !lease.Poll(&result) || result.StateValue != Managers::CookedCpuClosureCache::State::Ready || !result.Assets || !result.Assets->FindEnvironment(identity) || !manager.PollSceneResourcePreparation(scene, &preparation) || !preparation.Cooked || preparation.CookedManifest != result.Assets->ManifestDigest() || preparation.CookedScene != result.Assets->SceneUUID() || + preparation.State != RenderResourceManager::SceneResourcePreparationState::Preparing || !manager.GetSceneEnvironmentPreparation(scene, &environment) || environment.CookedSource.Valid() || !environment.Input.Sky.IsHDRI() || environment.Input.Sky.EnvironmentMap != identity.UUID) + { + return false; + } + m_lease = std::move(lease); + m_manager = &manager; + m_device = &device; + m_scene = scene; + m_identity = identity; + m_transfer_source = transfer_source; + m_state = State::Pending; + return true; + } + + CookedSceneEnvironmentBatch::State CookedSceneEnvironmentBatch::Step(uint8_t frame_index, uint8_t thread_index) + { + if (m_state != State::Pending) + { + return m_state; + } + Managers::CookedCpuClosureCache::Result result = {}; + RenderResourceManager::SceneResourcePreparationResult scene = {}; + if (!m_lease.Poll(&result) || result.StateValue != Managers::CookedCpuClosureCache::State::Ready || !result.Assets || m_device->IsDeviceLost.load(std::memory_order_acquire) || !m_manager->PollSceneResourcePreparation(m_scene, &scene) || !scene.Cooked || scene.State != RenderResourceManager::SceneResourcePreparationState::Preparing) + { + return m_state = State::Failed; + } + switch (m_upload.State()) + { + case CookedEnvironmentUpload::Phase::Empty: + { + Managers::CookedCpuClosureCache::Lease retained = {}; + if (!m_lease.Retain(&retained) || !m_upload.Capture(std::move(retained), m_identity)) + { + return m_state = State::Failed; + } + const auto* header = m_upload.Header(); + Specifications::TextureSpecification spec = {}; + spec.Width = header->FaceWidth; + spec.Height = header->FaceHeight; + spec.BytePerPixel = sizeof(float) * 4; + spec.LayerCount = header->LayerCount; + spec.MipLevelCount = header->MipCount; + spec.Format = Specifications::ImageFormat::R32G32B32A32_SFLOAT; + spec.IsCubemap = true; + spec.IsCookedAsset = true; + spec.IsUsageStorage = true; + spec.IsUsageTransferSource = m_transfer_source; + m_target = m_device->TryCreateTexture(spec, "cooked-scene-environment"); + if (!m_target.Valid() || !m_upload.Prepare(*m_manager)) + { + return m_state = State::Failed; + } + break; + } + case CookedEnvironmentUpload::Phase::Staging: + if (!m_upload.StageOne()) + { + return m_state = State::Failed; + } + break; + case CookedEnvironmentUpload::Phase::Staged: + (void) m_upload.Submit(frame_index, thread_index, m_target); + break; + case CookedEnvironmentUpload::Phase::Submitted: + { + const auto state = m_manager->GetTextureDecodeState(m_target); + if (state == RenderResourceManager::TextureDecodeState::Failed) + { + return m_state = State::Failed; + } + if (state != RenderResourceManager::TextureDecodeState::Succeeded || !m_manager->AppendCookedSceneEnvironment(m_scene, *result.Assets, m_upload)) + { + return m_state; + } + m_transferred = true; + m_state = State::Ready; + break; + } + default: + return m_state = State::Failed; + } + return m_state; + } + + void CookedSceneEnvironmentBatch::Reset() noexcept + { + m_upload.Reset(); + if (m_target.Valid() && !m_transferred) + { + m_manager->ReleaseTexture(m_target); + } + m_lease.Reset(); + m_manager = nullptr; + m_device = nullptr; + m_scene = {}; + m_identity = {}; + m_target = {}; + m_transfer_source = false; + m_transferred = false; + m_state = State::Empty; + } +} // namespace ZEngine::Rendering::Textures diff --git a/ZEngine/ZEngine/Rendering/Textures/CookedSceneEnvironmentBatch.h b/ZEngine/ZEngine/Rendering/Textures/CookedSceneEnvironmentBatch.h new file mode 100644 index 000000000..1904acb32 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Textures/CookedSceneEnvironmentBatch.h @@ -0,0 +1,58 @@ +#pragma once +#include + +namespace ZEngine::Hardwares +{ + struct VulkanDevice; +} + +namespace ZEngine::Rendering::Textures +{ + /// One cooked HDRI upload for a scene request. Initialize on an exclusive + /// worker, then transfer to the render owner for bounded Step calls. The + /// cache, manager and device outlive this object. + class CookedSceneEnvironmentBatch final + { + public: + enum class State : uint8_t + { + Empty, + Pending, + Ready, + Failed + }; + CookedSceneEnvironmentBatch() = default; + ~CookedSceneEnvironmentBatch(); + CookedSceneEnvironmentBatch(const CookedSceneEnvironmentBatch&) = delete; + CookedSceneEnvironmentBatch& operator=(const CookedSceneEnvironmentBatch&) = delete; + /// Failure preserves the lease and leaves this owner empty. transfer_source + /// is optional and permits an explicit GPU readback/transfer consumer. + [[nodiscard]] bool Initialize(Managers::CookedCpuClosureCache::Lease&&, const Cook::CookAssetIdentity&, RenderResourceManager&, Hardwares::VulkanDevice&, RenderResourceManager::SceneResourcePreparationId, bool transfer_source = false); + /// Ready means the scene owns the submitted source, not that the GPU + /// upload or sky bake has completed. + [[nodiscard]] State Step(uint8_t frame_index, uint8_t thread_index); + [[nodiscard]] State Status() const noexcept + { + return m_state; + } + [[nodiscard]] TextureHandle Texture() const noexcept + { + return m_target; + } + /// Untransferred targets retire through the resource manager. The scene + /// owns a transferred target and must be released separately on failure. + void Reset() noexcept; + + private: + Managers::CookedCpuClosureCache::Lease m_lease = {}; + CookedEnvironmentUpload m_upload = {}; + RenderResourceManager* m_manager = nullptr; + Hardwares::VulkanDevice* m_device = nullptr; + RenderResourceManager::SceneResourcePreparationId m_scene = {}; + Cook::CookAssetIdentity m_identity = {}; + TextureHandle m_target = {}; + bool m_transfer_source = false; + bool m_transferred = false; + State m_state = State::Empty; + }; +} // namespace ZEngine::Rendering::Textures diff --git a/ZEngine/ZEngine/Rendering/Textures/CookedSceneTextureBatch.cpp b/ZEngine/ZEngine/Rendering/Textures/CookedSceneTextureBatch.cpp new file mode 100644 index 000000000..c8d087d99 --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Textures/CookedSceneTextureBatch.cpp @@ -0,0 +1,284 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ZEngine::Rendering::Textures +{ + struct CookedSceneTextureBatch::Entry + { + Cook::CookAssetIdentity Identity = {}; + CookedTextureUpload Upload = {}; + TextureHandle Target = {}; + bool Transferred = false; + }; + + CookedSceneTextureBatch::~CookedSceneTextureBatch() + { + Reset(); + } + + bool CookedSceneTextureBatch::Initialize(Managers::CookedCpuClosureCache::Lease&& lease, Core::Memory::IAllocationRoute& route, RenderResourceManager& manager, Hardwares::VulkanDevice& device, RenderResourceManager::SceneResourcePreparationId scene, uint32_t max_textures) + { + Managers::CookedCpuClosureCache::Result result = {}; + RenderResourceManager::SceneResourcePreparationResult preparation = {}; + if (m_state != State::Empty || !scene || device.RRM != &manager || !lease.Poll(&result) || result.StateValue != Managers::CookedCpuClosureCache::State::Ready || !result.Assets || max_textures > RenderResourceManager::SceneCookedTextureCapacity || !manager.PollSceneResourcePreparation(scene, &preparation) || !preparation.Cooked || preparation.CookedManifest != result.Assets->ManifestDigest() || preparation.CookedScene != result.Assets->SceneUUID() || preparation.State != RenderResourceManager::SceneResourcePreparationState::Preparing) + { + return false; + } + uint32_t count = 0; + for (size_t i = 0; i < result.Assets->Size(); ++i) + { + const auto* version = result.Assets->VersionAt(i); + if (!version) + { + return false; + } + if (version->Identity.Kind == Cook::CookAssetKind::Texture && ++count > max_textures) + { + return false; + } + } + static_assert(alignof(Entry) <= alignof(std::max_align_t)); + if (count && size_t(count) > std::numeric_limits::max() / sizeof(Entry)) + { + return false; + } + auto* entries = count ? static_cast(route.Allocate(size_t(count) * sizeof(Entry))) : nullptr; + if (count && !entries) + { + return false; + } + uint32_t index = 0; + for (size_t i = 0; i < result.Assets->Size(); ++i) + { + const auto* version = result.Assets->VersionAt(i); + if (version->Identity.Kind == Cook::CookAssetKind::Texture) + { + new (entries + index) Entry{}; + entries[index++].Identity = version->Identity; + } + } + m_route = &route; + m_manager = &manager; + m_device = &device; + m_scene = scene; + m_entries = entries; + m_count = count; + m_lease = std::move(lease); + m_state = count ? State::Pending : State::Ready; + return true; + } + + CookedSceneTextureBatch::State CookedSceneTextureBatch::Step(uint8_t frame_index, uint8_t thread_index) + { + if (m_state != State::Pending) + { + return m_state; + } + const auto fail = [this](const char* reason) { + ZENGINE_CORE_ERROR("[CookedTextureBatch] {} submit={} transfer={} count={}", reason, m_submit_cursor, m_transfer_cursor, m_count) + return m_state = State::Failed; + }; + Managers::CookedCpuClosureCache::Result result = {}; + RenderResourceManager::SceneResourcePreparationResult scene = {}; + if (!m_lease.Poll(&result) || result.StateValue != Managers::CookedCpuClosureCache::State::Ready || !result.Assets || m_device->IsDeviceLost.load(std::memory_order_acquire) || !m_manager->PollSceneResourcePreparation(m_scene, &scene) || !scene.Cooked || scene.State != RenderResourceManager::SceneResourcePreparationState::Preparing) + { + return fail("closure, device, or scene is unavailable"); + } + if (m_submit_cursor < m_count) + { + auto& entry = m_entries[m_submit_cursor]; + switch (entry.Upload.State()) + { + case CookedTextureUpload::Phase::Empty: + { + const auto* image = result.Assets->FindTexture(entry.Identity); + Managers::CookedCpuClosureCache::Lease retained = {}; + if (!image || !m_lease.Retain(&retained) || !entry.Upload.Capture(std::move(retained), entry.Identity)) + return fail("decoded image or closure lease unavailable"); + Specifications::TextureSpecification spec = {}; + spec.Width = image->Description().Width; + spec.Height = image->Description().Height; + spec.BytePerPixel = 4; + spec.Format = Specifications::ImageFormat::R8G8B8A8_SRGB; + spec.IsCookedAsset = true; + VkResult status = VK_SUCCESS; + entry.Target = m_device->TryCreateTexture(spec, "cooked-scene-texture", 0, &status); + if (!entry.Target.Valid()) + { + ZENGINE_CORE_ERROR("[CookedTextureBatch] texture allocation status={} live_textures={} live_images={} texture_limit={}", static_cast(status), m_device->GlobalTextures.Size(), m_device->ImageBufferManager.Size(), m_device->MaxGlobalTexture) + return fail("texture allocation failed"); + } + if (!entry.Upload.Prepare(*m_manager, &status)) + { + ZENGINE_CORE_ERROR("[CookedTextureBatch] staging allocation status={}", static_cast(status)) + return fail("staging preparation failed"); + } + break; + } + case CookedTextureUpload::Phase::Staging: + { + // Each write stays bounded by UploadStaging::MaxSliceBytes. + // Use a small time and byte budget per render-owner visit + // so large textures do not require one frame per 64 KiB. + const auto start = std::chrono::steady_clock::now(); + const auto deadline = start + std::chrono::milliseconds(1); + constexpr uint32_t MaxWritesPerVisit = 16; + for (uint32_t writes = 0; writes < MaxWritesPerVisit && entry.Upload.State() == CookedTextureUpload::Phase::Staging; ++writes) + { + if (!entry.Upload.StageOne()) + return fail("staging write failed"); + ++m_staging_writes; + if (std::chrono::steady_clock::now() >= deadline) + break; + } + ++m_staging_visits; + const auto elapsed = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count()); + m_staging_max_ns = std::max(m_staging_max_ns, elapsed); + break; + } + case CookedTextureUpload::Phase::Staged: + if (entry.Upload.Submit(frame_index, thread_index, entry.Target)) + ++m_submit_cursor; + break; + default: + return fail("unexpected upload phase"); + } + return m_state; + } + auto& entry = m_entries[m_transfer_cursor]; + const auto ready = entry.Upload.Poll(); + if (ready == CookedTextureUpload::Readiness::Failed) + return fail("submitted texture readiness failed"); + if (ready != CookedTextureUpload::Readiness::Ready) + return m_state; + if (!m_manager->AppendCookedSceneTexture(m_scene, *result.Assets, entry.Upload)) + return fail("texture transfer to scene failed"); + entry.Transferred = true; + if (++m_transfer_cursor == m_count) + { + ZENGINE_CORE_INFO("[CookedTextureTiming] textures={} staging_visits={} staging_writes={} max_visit_ns={}", m_count, m_staging_visits, m_staging_writes, m_staging_max_ns) + m_state = State::Ready; + } + return m_state; + } + + CookedTextureUpload* CookedSceneTextureBatch::Find(const Cook::CookAssetIdentity& identity) noexcept + { + if (identity.Kind != Cook::CookAssetKind::Texture || !m_count) + { + return nullptr; + } + auto* found = std::lower_bound(m_entries, m_entries + m_count, identity, [](const Entry& entry, const Cook::CookAssetIdentity& key) { return entry.Identity < key; }); + return found != m_entries + m_count && found->Identity == identity ? &found->Upload : nullptr; + } + + CookedSceneTextureBatch::State CookedSceneTextureBatch::StepMaterials() + { + if (m_material_state != State::Pending) + { + return m_material_state; + } + const auto fail = [this](const char* reason) { + ZENGINE_CORE_ERROR("[CookedMaterialBatch] {} cursor={}", reason, m_material_cursor) + return m_material_state = State::Failed; + }; + if (m_state == State::Failed || m_state == State::Empty) + { + return fail("texture batch unavailable"); + } + if (m_state != State::Ready) + { + return State::Pending; + } + Managers::CookedCpuClosureCache::Result result = {}; + RenderResourceManager::SceneResourcePreparationResult scene = {}; + if (!m_lease.Poll(&result) || result.StateValue != Managers::CookedCpuClosureCache::State::Ready || !result.Assets || !m_manager->PollSceneResourcePreparation(m_scene, &scene) || !scene.Cooked || scene.State != RenderResourceManager::SceneResourcePreparationState::Preparing) + { + return fail("closure or scene unavailable"); + } + if (m_material_cursor == result.Assets->Size()) + { + return m_material_state = State::Ready; + } + const auto* version = result.Assets->VersionAt(m_material_cursor); + if (!version) + { + return fail("asset version unavailable"); + } + if (version->Identity.Kind == Cook::CookAssetKind::Material) + { + const auto* material = result.Assets->FindMaterial(version->Identity); + if (!material) + { + return fail("material unavailable"); + } + std::array uploads = {}; + size_t count = 0; + for (const auto& map : material->Textures) + { + if (map.UUID.is_nil()) + { + continue; + } + auto* upload = Find({Cook::CookAssetKind::Texture, map.UUID}); + if (!upload) + { + return fail("material texture unavailable"); + } + if (std::find(uploads.begin(), uploads.begin() + count, upload) == uploads.begin() + count) + { + uploads[count++] = upload; + } + } + std::sort(uploads.begin(), uploads.begin() + count, [](const auto* left, const auto* right) { return left->Identity() < right->Identity(); }); + Materials::CookedMaterialVersion prepared = {}; + const auto state = Materials::PrepareCookedMaterial(*result.Assets, version->Identity, {uploads.data(), count}, *m_manager, &prepared); + if (state == SceneTextureDescriptorState::Pending) + { + return State::Pending; + } + if (state != SceneTextureDescriptorState::Ready || !m_manager->AppendCookedSceneMaterial(m_scene, *result.Assets, prepared)) + { + return fail("material preparation or scene transfer failed"); + } + } + ++m_material_cursor; + if (m_material_cursor == result.Assets->Size()) + { + m_material_state = State::Ready; + } + return m_material_state; + } + + void CookedSceneTextureBatch::Reset() noexcept + { + for (uint32_t i = 0; i < m_count; ++i) + { + auto& entry = m_entries[i]; + entry.Upload.Reset(); + if (entry.Target.Valid() && !entry.Transferred) + m_manager->ReleaseTexture(entry.Target); + entry.~Entry(); + } + if (m_entries) + m_route->Free(m_entries); + m_lease.Reset(); + m_route = nullptr; + m_manager = nullptr; + m_device = nullptr; + m_scene = {}; + m_entries = nullptr; + m_count = m_submit_cursor = m_transfer_cursor = 0; + m_staging_visits = m_staging_writes = m_staging_max_ns = 0; + m_material_cursor = 0; + m_material_state = State::Pending; + m_state = State::Empty; + } +} // namespace ZEngine::Rendering::Textures diff --git a/ZEngine/ZEngine/Rendering/Textures/CookedSceneTextureBatch.h b/ZEngine/ZEngine/Rendering/Textures/CookedSceneTextureBatch.h new file mode 100644 index 000000000..30262969e --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Textures/CookedSceneTextureBatch.h @@ -0,0 +1,74 @@ +#pragma once +#include +#include + +namespace ZEngine::Hardwares +{ + struct VulkanDevice; +} + +namespace ZEngine::Rendering::Textures +{ + /// Transaction for the texture portion of a cooked scene. Initialize on an + /// exclusive worker, then transfer ownership to the render owner for Step. + /// One Step performs at most sixteen 64 KiB staging writes, stopping after + /// a 1 ms deadline between writes, or performs one admission. + /// The route, cache, manager and device outlive this owner. The host keeps + /// rendering/processing graph acquire submissions while Pending. + class CookedSceneTextureBatch final + { + public: + enum class State : uint8_t + { + Empty, + Pending, + Ready, + Failed + }; + CookedSceneTextureBatch() = default; + ~CookedSceneTextureBatch(); + CookedSceneTextureBatch(const CookedSceneTextureBatch&) = delete; + CookedSceneTextureBatch& operator=(const CookedSceneTextureBatch&) = delete; + + /// Consumes the ready closure lease only after bounded allocation and + /// complete admission. No scene or output state changes on failure. + [[nodiscard]] bool Initialize(Managers::CookedCpuClosureCache::Lease&&, Core::Memory::IAllocationRoute&, RenderResourceManager&, Hardwares::VulkanDevice&, RenderResourceManager::SceneResourcePreparationId, uint32_t max_textures); + [[nodiscard]] State Step(uint8_t frame_index, uint8_t thread_index); + /// After textures are ready, retain one closure material per call in + /// the same scene request. Non-material entries also consume a step. + /// Pending includes texture acquisition and descriptor backpressure. + [[nodiscard]] State StepMaterials(); + /// Borrow for material preparation while this batch remains alive. + [[nodiscard]] CookedTextureUpload* Find(const Cook::CookAssetIdentity&) noexcept; + [[nodiscard]] State Status() const noexcept + { + return m_state; + } + [[nodiscard]] State MaterialStatus() const noexcept + { + return m_material_state; + } + /// Releases untransferred destinations through normal GPU retirement. + /// The scene owns successfully transferred destinations. The caller must + /// cancel/release the scene request after failure or cancellation. + void Reset() noexcept; + + private: + struct Entry; + Core::Memory::IAllocationRoute* m_route = nullptr; + RenderResourceManager* m_manager = nullptr; + Hardwares::VulkanDevice* m_device = nullptr; + Managers::CookedCpuClosureCache::Lease m_lease = {}; + RenderResourceManager::SceneResourcePreparationId m_scene = {}; + Entry* m_entries = nullptr; + uint32_t m_count = 0; + uint32_t m_submit_cursor = 0; + uint32_t m_transfer_cursor = 0; + size_t m_material_cursor = 0; + uint64_t m_staging_visits = 0; + uint64_t m_staging_writes = 0; + uint64_t m_staging_max_ns = 0; + State m_material_state = State::Pending; + State m_state = State::Empty; + }; +} // namespace ZEngine::Rendering::Textures diff --git a/ZEngine/ZEngine/Rendering/Textures/CookedTextureUpload.cpp b/ZEngine/ZEngine/Rendering/Textures/CookedTextureUpload.cpp new file mode 100644 index 000000000..16e45868c --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Textures/CookedTextureUpload.cpp @@ -0,0 +1,181 @@ +#include +#include +#include + +namespace ZEngine::Rendering::Textures +{ + bool CookedTextureUpload::Capture(Managers::CookedCpuClosureCache::Lease&& lease, const Cook::CookAssetIdentity& identity) + { + Managers::CookedCpuClosureCache::Result result = {}; + if (m_phase != Phase::Empty || identity.Kind != Cook::CookAssetKind::Texture || !identity.IsValid() || !lease.Poll(&result) || result.StateValue != Managers::CookedCpuClosureCache::State::Ready || !result.Assets) + { + return false; + } + const auto* version = result.Assets->FindVersion(identity); + const auto* image = result.Assets->FindTexture(identity); + if (!version || !image) + { + return false; + } + const auto& description = image->Description(); + const size_t width = description.Width; + const size_t height = description.Height; + if (!width || !height || width > std::numeric_limits::max() / height || width * height > std::numeric_limits::max() / 4 || image->Pixels().size() != width * height * 4) + { + return false; + } + m_identity = identity; + m_digest = version->ObjectDigest; + m_image = image; + m_lease = std::move(lease); + m_phase = Phase::SourceReady; + return true; + } + + bool CookedTextureUpload::Prepare(RenderResourceManager& manager, VkResult* status) + { + if (m_phase != Phase::SourceReady) + { + if (status) + { + *status = VK_ERROR_INITIALIZATION_FAILED; + } + return false; + } + if (!manager.PrepareUploadStaging(m_image->Pixels().size(), &m_staging, status)) + { + return false; + } + m_manager = &manager; + m_phase = Phase::Staging; + return true; + } + + bool CookedTextureUpload::StageOne(VkResult* status) + { + if (m_phase != Phase::Staging || !m_image) + { + if (status) + { + *status = VK_ERROR_INITIALIZATION_FAILED; + } + return false; + } + const auto pixels = m_image->Pixels(); + const size_t offset = static_cast(m_staging.Written()); + if (offset >= pixels.size()) + { + return false; + } + size_t consumed = 0; + if (!m_staging.Append({pixels.data() + offset, pixels.size() - offset}, &consumed, status)) + { + return false; + } + if (m_staging.Ready()) + { + m_phase = Phase::Staged; + } + return true; + } + + bool CookedTextureUpload::Submit(uint8_t frame_index, uint8_t thread_index, TextureHandle target) + { + if (m_phase != Phase::Staged || !m_manager || !m_image || !target.Valid() || target.Index >= INVALID_MAP_HANDLE) + { + return false; + } + const auto* texture = m_manager->GetTexture(target); + const auto& description = m_image->Description(); + if (!texture || texture->Width != description.Width || texture->Height != description.Height || texture->BytePerPixel != 4 || texture->Specification.Format != Specifications::ImageFormat::R8G8B8A8_SRGB || texture->Specification.Depth != 1 || texture->Specification.LayerCount != 1 || texture->Specification.MipLevelCount != 1 || texture->Specification.Is3D || texture->Specification.IsCubemap || texture->Specification.IsAliasable || !texture->Specification.IsUsageSampled || !texture->Specification.IsUsageTransfert) + { + return false; + } + if (!m_manager->UploadPreparedTextureBuffer(frame_index, thread_index, target, m_staging).Valid()) + { + return false; + } + m_binding = {m_identity, m_digest, target}; + // UploadPreparedTextureBuffer transfers the fully written staging buffer + // to the upload owner. No later phase reads decoded pixels, so retaining + // one closure lease per submitted texture would unnecessarily exhaust the + // bounded per-closure lease budget before large batches can finish. + m_image = nullptr; + m_lease.Reset(); + m_phase = Phase::Submitted; + return true; + } + + CookedTextureUpload::Readiness CookedTextureUpload::Poll() + { + m_producer_descriptor_ready = false; + m_consumer_ready = false; + if (m_phase == Phase::Empty || m_phase == Phase::Failed) + { + return Readiness::Failed; + } + if (m_phase != Phase::Submitted) + { + return Readiness::Pending; + } + const auto state = m_manager->PrepareSubmittedTextureVersion(m_binding.Handle, &m_texture_pin, &m_descriptor_publication); + if (state == SceneTextureDescriptorState::Failed) + { + m_phase = Phase::Failed; + return Readiness::Failed; + } + if (state == SceneTextureDescriptorState::Pending) + { + return Readiness::Pending; + } + m_producer_descriptor_ready = true; + const auto acquire = m_manager->PollSubmittedTextureAcquire(m_texture_pin); + if (acquire == SceneTextureDescriptorState::Failed) + { + m_phase = Phase::Failed; + m_producer_descriptor_ready = false; + return Readiness::Failed; + } + if (acquire == SceneTextureDescriptorState::Pending) + { + return Readiness::AwaitingConsumer; + } + m_consumer_ready = true; + return Readiness::Ready; + } + + const Materials::CookedTextureBinding* CookedTextureUpload::ProducerDescriptorBinding() + { + if (m_phase != Phase::Submitted || !m_producer_descriptor_ready || !m_manager->IsSubmittedTextureDescriptorCurrent(m_texture_pin, m_descriptor_publication)) + { + return nullptr; + } + return &m_binding; + } + + RenderResourceManager::TextureVersionPin CookedTextureUpload::RetainPreparedVersion() + { + return ProducerDescriptorBinding() ? m_manager->RetainTextureVersion(m_texture_pin) : RenderResourceManager::TextureVersionPin{}; + } + + const Materials::CookedTextureBinding* CookedTextureUpload::ReadyBinding() + { + return m_consumer_ready && ProducerDescriptorBinding() && m_manager->PollSubmittedTextureAcquire(m_texture_pin) == SceneTextureDescriptorState::Ready ? &m_binding : nullptr; + } + + void CookedTextureUpload::Reset() noexcept + { + m_texture_pin.Reset(); + m_staging.Reset(); + m_lease.Reset(); + m_image = nullptr; + m_manager = nullptr; + m_identity = {}; + m_digest = {}; + m_binding = {}; + m_descriptor_publication = 0; + m_producer_descriptor_ready = false; + m_consumer_ready = false; + m_phase = Phase::Empty; + } +} // namespace ZEngine::Rendering::Textures diff --git a/ZEngine/ZEngine/Rendering/Textures/CookedTextureUpload.h b/ZEngine/ZEngine/Rendering/Textures/CookedTextureUpload.h new file mode 100644 index 000000000..8e50135de --- /dev/null +++ b/ZEngine/ZEngine/Rendering/Textures/CookedTextureUpload.h @@ -0,0 +1,101 @@ +#pragma once +#include +#include +#include + +namespace ZEngine::Rendering::Textures +{ + /// One unpublished cooked texture's source and staging transaction. The + /// closure lease keeps decoded pixels alive through accepted staging submission; + /// identity and digest are copied for the remaining GPU lifetime. Each + /// instance is exclusively owned; externally synchronize thread handoffs. + /// The cache, resource manager and device outlive the transaction. + class CookedTextureUpload final + { + public: + enum class Phase : uint8_t + { + Empty, + SourceReady, + Staging, + Staged, + Submitted, + Failed + }; + enum class Readiness : uint8_t + { + Pending, + AwaitingConsumer, + Ready, + Failed + }; + + CookedTextureUpload() = default; + ~CookedTextureUpload() = default; + CookedTextureUpload(const CookedTextureUpload&) = delete; + CookedTextureUpload& operator=(const CookedTextureUpload&) = delete; + + /// Consumes a ready closure lease only on success. Source assets and + /// registries are never consulted; the selected version is immutable. + [[nodiscard]] bool Capture(Managers::CookedCpuClosureCache::Lease&& lease, const Cook::CookAssetIdentity& identity); + /// Render-owner allocation. Failure preserves the source for retry. + [[nodiscard]] bool Prepare(RenderResourceManager& manager, VkResult* status = nullptr); + /// Copies one bounded slice. A failed write does not advance the cursor. + [[nodiscard]] bool StageOne(VkResult* status = nullptr); + /// Render-owner submission into a caller-owned, compatible image reserved + /// exclusively for this version until it is pinned or retired. A + /// refusal preserves the staged data. The returned binding represents + /// submission only: it does not establish GPU, descriptor, or consumer + /// readiness, nor does it retain the target image. + [[nodiscard]] bool Submit(uint8_t frame_index, uint8_t thread_index, TextureHandle target); + /// Render-owner poll. Ready additionally requires completion of an + /// accepted graphics-family acquire. It is texture readiness, not a + /// claim that the rest of the scene is ready for publication. + [[nodiscard]] Readiness Poll(); + /// A clone of the retained version, only while its descriptor is current. + /// Its caller must retain it through every CPU and GPU consumer. + [[nodiscard]] RenderResourceManager::TextureVersionPin RetainPreparedVersion(); + void Reset() noexcept; + + [[nodiscard]] Phase State() const noexcept + { + return m_phase; + } + [[nodiscard]] VkDeviceSize StagedBytes() const noexcept + { + return m_staging.Written(); + } + [[nodiscard]] const Materials::CookedTextureBinding* SubmittedBinding() const noexcept + { + return m_phase == Phase::Submitted ? &m_binding : nullptr; + } + [[nodiscard]] const Materials::CookedTextureBinding* ProducerDescriptorBinding(); + [[nodiscard]] const Materials::CookedTextureBinding* ReadyBinding(); + [[nodiscard]] const Cook::CookAssetIdentity& Identity() const noexcept + { + return m_identity; + } + [[nodiscard]] const Cook::Digest256& Digest() const noexcept + { + return m_digest; + } + [[nodiscard]] bool BelongsTo(const RenderResourceManager& manager) const noexcept + { + return m_manager == &manager; + } + + private: + Managers::CookedCpuClosureCache::Lease m_lease = {}; + const Importers::ImageCodec::DecodedImage* m_image = nullptr; + RenderResourceManager* m_manager = nullptr; + RenderResourceManager::UploadStaging m_staging = {}; + RenderResourceManager::TextureVersionPin m_texture_pin = {}; + Cook::CookAssetIdentity m_identity = {}; + Cook::Digest256 m_digest = {}; + Materials::CookedTextureBinding m_binding = {}; + uint64_t m_descriptor_publication = 0; + bool m_producer_descriptor_ready = false; + bool m_consumer_ready = false; + Phase m_phase = Phase::Empty; + }; +} // namespace ZEngine::Rendering::Textures diff --git a/ZEngine/ZEngine/Rendering/Textures/Texture.h b/ZEngine/ZEngine/Rendering/Textures/Texture.h index 568f1d7ae..c2f6c866b 100644 --- a/ZEngine/ZEngine/Rendering/Textures/Texture.h +++ b/ZEngine/ZEngine/Rendering/Textures/Texture.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace ZEngine::Hardwares { @@ -24,7 +25,41 @@ namespace ZEngine::Rendering::Textures Specifications::TextureSpecification Specification = {}; Helpers::Handle BufferHandle = {}; - void Dispose(); + // The existing buffer-to-image copy covers mip zero, all layers and depth. + // BufferSize includes every mip and is not the minimum copy size. + [[nodiscard]] bool ContainsBaseMip(VkDeviceSize size, VkDeviceSize* base_mip_bytes = nullptr) const noexcept + { + // CopyBufferToImage currently records a color-aspect copy. The + // declared texel size must match the Vulkan format it will copy. + const auto format = Specification.Format; + const auto index = static_cast(format); + const bool color = (format >= Specifications::ImageFormat::R8G8B8A8_UNORM && format <= Specifications::ImageFormat::R32G32B32_SFLOAT) || format == Specifications::ImageFormat::R32_UINT; + if (!color || BytePerPixel != Specifications::BytePerChannelMap[index]) + { + return false; + } + VkDeviceSize required = 1; + const uint32_t dimensions[] = {Width, Height, Specification.Depth, Specification.LayerCount, BytePerPixel}; + for (const uint32_t dimension : dimensions) + { + if (!dimension || required > std::numeric_limits::max() / dimension) + { + return false; + } + required *= dimension; + } + if (size < required || size > BufferSize) + { + return false; + } + if (base_mip_bytes) + { + *base_mip_bytes = required; + } + return true; + } + + void Dispose(); }; using TextureHandle = Helpers::Handle; @@ -41,4 +76,4 @@ namespace ZEngine::Helpers m_memory[i].Dispose(); } } -} // namespace ZEngine::Helpers \ No newline at end of file +} // namespace ZEngine::Helpers diff --git a/ZEngine/ZEngine/Windows/GameWindow.cpp b/ZEngine/ZEngine/Windows/GameWindow.cpp index 2f55c1e1e..9f5342f93 100644 --- a/ZEngine/ZEngine/Windows/GameWindow.cpp +++ b/ZEngine/ZEngine/Windows/GameWindow.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -13,9 +14,14 @@ // and returns immediately; on_complete fires later on the main run loop. // Implemented in Windows/Platform/MacOSAppIcon.mm extern "C" void ZEngineSetDockIcon(const char* path); +extern "C" int ZEngineConfigureMetalLayerDelegate(void* window); +#if ZENGINE_PROFILING +extern "C" int ZEngineWindowOcclusionVisible(void* window); +#endif #endif #include #include +#include #include #include #include @@ -113,6 +119,7 @@ namespace ZEngine::Windows } glfwWindowHint(GLFW_CLIENT_API, GLFW_NO_API); + glfwWindowHint(GLFW_RESIZABLE, cfg.Resizable ? GLFW_TRUE : GLFW_FALSE); glfwSetErrorCallback([](int error, const char* description) { ZENGINE_CORE_CRITICAL("{}", description) @@ -200,12 +207,51 @@ namespace ZEngine::Windows glfwSetCursorPosCallback(m_native_window, GameWindow::__OnGlfwCursorMoved); glfwSetCharCallback(m_native_window, GameWindow::__OnGlfwTextInputRaised); - glfwMaximizeWindow(m_native_window); + if (cfg.Maximize) + glfwMaximizeWindow(m_native_window); +#if defined(__APPLE__) && ZENGINE_PROFILING + if (cfg.WaitForVisibleTestWindow) + { + // Test-only gate: pump platform events before the timed scene + // admission, rather than crediting hidden-window frames as visible. + const double deadline = glfwGetTime() + 15.0; + while (ZEngineWindowOcclusionVisible(glfwGetCocoaWindow(m_native_window)) != 1 && glfwGetTime() < deadline && !glfwWindowShouldClose(m_native_window)) + glfwWaitEventsTimeout(0.05); + if (ZEngineWindowOcclusionVisible(glfwGetCocoaWindow(m_native_window)) != 1) + { + ZENGINE_CORE_ERROR("Timed out waiting for the test window to become visible") + ZENGINE_EXIT_FAILURE(); + } + } + m_previous_occlusion = ZEngineWindowOcclusionVisible(glfwGetCocoaWindow(m_native_window)); + m_log_test_occlusion = cfg.WaitForVisibleTestWindow; + if (m_log_test_occlusion) + { + const auto now = std::chrono::steady_clock::now().time_since_epoch(); + const auto timestamp_ns = std::chrono::duration_cast(now).count(); + ZENGINE_CORE_INFO("[CookedWindowVisibility] timestamp_ns={} visible={}", timestamp_ns, m_previous_occlusion < 0 ? 2 : m_previous_occlusion) + } + ZENGINE_PROFILE_VALUE("SceneLoad.Main.WindowOcclusionVisible", static_cast(m_previous_occlusion < 0 ? 2 : m_previous_occlusion)); +#endif } void GameWindow::PollEvent() { glfwPollEvents(); +#if defined(__APPLE__) && ZENGINE_PROFILING + const int visible = ZEngineWindowOcclusionVisible(glfwGetCocoaWindow(m_native_window)); + if (visible != m_previous_occlusion) + { + m_previous_occlusion = visible; + if (m_log_test_occlusion) + { + const auto now = std::chrono::steady_clock::now().time_since_epoch(); + const auto timestamp_ns = std::chrono::duration_cast(now).count(); + ZENGINE_CORE_INFO("[CookedWindowVisibility] timestamp_ns={} visible={}", timestamp_ns, visible < 0 ? 2 : visible) + } + ZENGINE_PROFILE_VALUE("SceneLoad.Main.WindowOcclusionVisible", static_cast(visible < 0 ? 2 : visible)); + } +#endif } float GameWindow::GetTime() @@ -422,6 +468,15 @@ namespace ZEngine::Windows VkInstance vkInstance = reinterpret_cast(instance); VkSurfaceKHR* pSurface = reinterpret_cast(out_window_surface); VkResult result = glfwCreateWindowSurface(vkInstance, m_native_window, nullptr, pSurface); +#if defined(__APPLE__) + if (result == VK_SUCCESS) + { + // GLFW creates a layer explicitly. MoltenVK recommends that its + // hosting view be the delegate for correct display timing. + const int delegate_state = ZEngineConfigureMetalLayerDelegate(glfwGetCocoaWindow(m_native_window)); + ZENGINE_CORE_INFO("[MacOSSurface] metal_layer_delegate_state={}", delegate_state) + } +#endif return (result == VK_SUCCESS); } diff --git a/ZEngine/ZEngine/Windows/GameWindow.h b/ZEngine/ZEngine/Windows/GameWindow.h index 9d8d6a831..8f633feae 100644 --- a/ZEngine/ZEngine/Windows/GameWindow.h +++ b/ZEngine/ZEngine/Windows/GameWindow.h @@ -63,6 +63,10 @@ namespace ZEngine::Windows private: ZEngine::Windows::FileDialogRequest FileDialogAsync(bool save, std::span type_filters, std::string_view default_dir, std::string_view filename, std::string_view message); GLFWwindow* m_native_window = nullptr; +#if defined(__APPLE__) && ZENGINE_PROFILING + int m_previous_occlusion = -2; + bool m_log_test_occlusion = false; +#endif }; } // namespace ZEngine::Windows diff --git a/ZEngine/ZEngine/Windows/Platform/MacOSAppIcon.mm b/ZEngine/ZEngine/Windows/Platform/MacOSAppIcon.mm index c56f2d226..f0beb6281 100644 --- a/ZEngine/ZEngine/Windows/Platform/MacOSAppIcon.mm +++ b/ZEngine/ZEngine/Windows/Platform/MacOSAppIcon.mm @@ -1,4 +1,5 @@ #import +#import extern "C" void ZEngineSetDockIcon(const char* path) { @@ -9,3 +10,28 @@ [[NSApplication sharedApplication] setApplicationIconImage:image]; } } + +extern "C" int ZEngineWindowOcclusionVisible(void* window) +{ + if (!window) + return -1; + NSWindow* native_window = (__bridge NSWindow*) window; + return (native_window.occlusionState & NSWindowOcclusionStateVisible) != 0 ? 1 : 0; +} + +extern "C" int ZEngineConfigureMetalLayerDelegate(void* window) +{ + if (!window) + return 0; + NSWindow* native_window = (__bridge NSWindow*) window; + NSView* view = native_window.contentView; + CALayer* layer = view.layer; + if (![layer isKindOfClass:[CAMetalLayer class]]) + return 0; + if (!layer.delegate) + { + layer.delegate = (id) view; + return 1; + } + return layer.delegate == (id) view ? 2 : 3; +} diff --git a/ZEngine/ZEngine/Windows/WindowConfiguration.h b/ZEngine/ZEngine/Windows/WindowConfiguration.h index 8ca3463f1..7e6687d8f 100644 --- a/ZEngine/ZEngine/Windows/WindowConfiguration.h +++ b/ZEngine/ZEngine/Windows/WindowConfiguration.h @@ -5,9 +5,12 @@ namespace ZEngine::Windows { struct WindowConfiguration { - uint32_t Width = 1500; - uint32_t Height = 800; - bool EnableVsync = true; + uint32_t Width = 1500; + uint32_t Height = 800; + bool EnableVsync = true; + bool Resizable = true; + bool Maximize = true; + bool WaitForVisibleTestWindow = false; Core::Containers::String Title; }; ZDEFINE_PTR(WindowConfiguration); diff --git a/ZEngine/tests/CMakeLists.txt b/ZEngine/tests/CMakeLists.txt index fff72c204..b51de25df 100644 --- a/ZEngine/tests/CMakeLists.txt +++ b/ZEngine/tests/CMakeLists.txt @@ -42,14 +42,14 @@ target_sources(ZEngineTests PRIVATE ${TEST_SOURCES}) target_sources(ZEngineTests PRIVATE Cook/Process/SceneCookCustomSchema.cpp) # Use the reviewed contract bytes themselves, also in installed/headless tests. -set(COOK_MANIFEST_GOLDEN_PATH "${CMAKE_SOURCE_DIR}/docs/design/future-plan/scene-cooker-contracts/manifest-v1.golden.json") +set(COOK_MANIFEST_GOLDEN_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Cook/Fixtures/manifest-v1.golden.json") set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${COOK_MANIFEST_GOLDEN_PATH}") file(READ "${COOK_MANIFEST_GOLDEN_PATH}" COOK_MANIFEST_GOLDEN) -set(COOK_MANIFEST_NEGATIVE_PATH "${CMAKE_SOURCE_DIR}/docs/design/future-plan/scene-cooker-contracts/manifest-v1.negative-cases.json") +set(COOK_MANIFEST_NEGATIVE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Cook/Fixtures/manifest-v1.negative-cases.json") set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${COOK_MANIFEST_NEGATIVE_PATH}") file(READ "${COOK_MANIFEST_NEGATIVE_PATH}" COOK_MANIFEST_NEGATIVE) configure_file(Cook/CookManifestGolden.h.in "generated/CookManifestGolden.h" @ONLY) -set_property(SOURCE Cook/CookManifestWriter_test.cpp Cook/CookManifestReader_test.cpp APPEND PROPERTY +set_property(SOURCE Cook/CookManifestWriter_test.cpp Cook/CookManifestReader_test.cpp Cook/CookedPackageSnapshot_test.cpp APPEND PROPERTY INCLUDE_DIRECTORIES "${CMAKE_CURRENT_BINARY_DIR}/generated") # Embed the real production GLSL, not a second test shader. Installed test diff --git a/ZEngine/tests/Cook/CookedObjectSnapshot_test.cpp b/ZEngine/tests/Cook/CookedObjectSnapshot_test.cpp new file mode 100644 index 000000000..231c56108 --- /dev/null +++ b/ZEngine/tests/Cook/CookedObjectSnapshot_test.cpp @@ -0,0 +1,192 @@ +#include +#include +#include +#include +#include +#include "CookObjectTestSupport.h" + +using ZEngine::Managers::CookedObjectSnapshot; + +namespace ZEngine::Cook +{ + namespace VFS = Core::VFS; + + namespace + { + struct SnapshotRoute final : Core::Memory::IAllocationRoute + { + alignas(std::max_align_t) std::array Storage = {}; + std::atomic* CancelOnAllocate = nullptr; + bool InUse = false; + bool Refuse = false; + uint32_t Allocations = 0; + uint32_t Frees = 0; + + void* Allocate(size_t bytes) noexcept override + { + ++Allocations; + if (Refuse || InUse || bytes > Storage.size()) + { + return nullptr; + } + InUse = true; + if (CancelOnAllocate) + { + CancelOnAllocate->store(true, std::memory_order_release); + } + return Storage.data(); + } + void* Reallocate(void*, size_t, size_t) noexcept override + { + return nullptr; + } + void Free(void* pointer) noexcept override + { + if (pointer) + { + EXPECT_EQ(pointer, Storage.data()); + EXPECT_TRUE(InUse); + InUse = false; + ++Frees; + } + } + }; + } // namespace + + class CookedObjectSnapshotTest : public testing::Test + { + protected: + Tests::ObjectStore Store = {}; + SnapshotRoute Route = {}; + CookObjectDescriptor Object = {}; + CookedObjectSnapshot Snapshot = {}; + + void SetUp() override + { + std::array bytes = {}; + for (size_t i = 0; i < bytes.size(); ++i) + { + bytes[i] = static_cast(i); + } + Object = Store.Entries[Store.Add({bytes.data(), bytes.size()}, "zescenebin")].Object; + } + void TearDown() override + { + Snapshot.Reset(); + EXPECT_FALSE(Route.InUse); + } + VFS::VFSResult Capture(size_t max_bytes = 128, const std::atomic* cancelled = nullptr) + { + return CookedObjectSnapshot::Capture(Store.Reader(), Object, "zescenebin", Route, max_bytes, &Snapshot, cancelled); + } + }; + + TEST_F(CookedObjectSnapshotTest, RetainsExactlyTheVerifiedBytesAfterTheSourceChanges) + { + Store.Entries[0].File.ReadLimit = 7; + ASSERT_TRUE(Capture().Succeeded()); + ASSERT_EQ(Snapshot.Bytes().size(), 128u); + EXPECT_EQ(Store.Entries[0].File.Closes, 1u); + EXPECT_EQ(Store.Entries[0].File.Stats, 2u); + Store.Entries[0].Bytes[0] = 255; + EXPECT_EQ(Snapshot.Bytes()[0], 0u); + EXPECT_EQ(Snapshot.Bytes()[127], 127u); + EXPECT_EQ(Capture().Error(), VFS::VFSError::InvalidPath); + EXPECT_EQ(Store.Opens, 1u); + CookedObjectSnapshot moved = std::move(Snapshot); + EXPECT_TRUE(Snapshot.Bytes().empty()); + EXPECT_EQ(moved.Bytes()[127], 127u); + moved.Reset(); + EXPECT_EQ(Route.Frees, 1u); + } + + TEST_F(CookedObjectSnapshotTest, RejectsInvalidPathsAndOverBudgetBeforeOpening) + { + EXPECT_EQ(Capture(127).Error(), VFS::VFSError::SizeLimitExceeded); + auto invalid = Object; + invalid.Digest.Bytes[0] ^= 1; + EXPECT_EQ(CookedObjectSnapshot::Capture(Store.Reader(), invalid, "zescenebin", Route, 128, &Snapshot).Error(), VFS::VFSError::Corrupted); + EXPECT_EQ(CookedObjectSnapshot::Capture(Store.Reader(), Object, "png", Route, 128, &Snapshot).Error(), VFS::VFSError::Corrupted); + EXPECT_EQ(Store.Opens, 0u); + EXPECT_EQ(Route.Allocations, 0u); + } + + TEST_F(CookedObjectSnapshotTest, RejectsCorruptionShortReadsAndFailuresWithoutRetainingStorage) + { + auto& file = Store.Entries[0].File; + Store.Entries[0].Bytes[0] ^= 1; + EXPECT_EQ(Capture().Error(), VFS::VFSError::Corrupted); + Store.Entries[0].Bytes[0] ^= 1; + file.EOFAt = Object.Size - 1; + EXPECT_EQ(Capture().Error(), VFS::VFSError::Corrupted); + file.EOFAt = UINT64_MAX; + file.Overreport = true; + EXPECT_EQ(Capture().Error(), VFS::VFSError::Corrupted); + file.Overreport = false; + file.ChangedSize = Object.Size + 1; + EXPECT_EQ(Capture().Error(), VFS::VFSError::Corrupted); + file.ChangedSize = UINT64_MAX; + file.ReadFailureAt = 0; + file.ReadError = VFS::VFSError::PermissionDenied; + file.CloseError = VFS::VFSError::IOError; + EXPECT_EQ(Capture().Error(), VFS::VFSError::PermissionDenied); + file.ReadFailureAt = UINT64_MAX; + EXPECT_EQ(Capture().Error(), VFS::VFSError::IOError); + EXPECT_TRUE(Snapshot.Bytes().empty()); + EXPECT_EQ(file.Closes, 6u); + EXPECT_EQ(Route.Allocations, Route.Frees); + } + + TEST_F(CookedObjectSnapshotTest, CancellationAndAllocationRefusalCloseAndReleaseOwnedWork) + { + std::atomic cancelled = true; + EXPECT_EQ(Capture(128, &cancelled).Error(), VFS::VFSError::Cancelled); + EXPECT_EQ(Store.Opens, 0u); + cancelled.store(false); + Route.CancelOnAllocate = &cancelled; + EXPECT_EQ(Capture(128, &cancelled).Error(), VFS::VFSError::Cancelled); + EXPECT_EQ(Store.Entries[0].File.Closes, 1u); + EXPECT_FALSE(Route.InUse); + cancelled.store(false); + Route.CancelOnAllocate = nullptr; + Route.Refuse = true; + EXPECT_EQ(Capture().Error(), VFS::VFSError::OutOfMemory); + EXPECT_EQ(Store.Entries[0].File.Closes, 2u); + EXPECT_TRUE(Snapshot.Bytes().empty()); + } + + TEST_F(CookedObjectSnapshotTest, OpenAndStatFailuresReleaseResourcesAndAllowRetry) + { + auto& entry = Store.Entries[0]; + auto& file = entry.File; + entry.OpenError = VFS::VFSError::PermissionDenied; + EXPECT_EQ(Capture().Error(), VFS::VFSError::PermissionDenied); + EXPECT_EQ(Route.Allocations, 0u); + EXPECT_EQ(file.Closes, 0u); + entry.OpenError = VFS::VFSError::OK; + entry.NullHandle = true; + EXPECT_EQ(Capture().Error(), VFS::VFSError::IOError); + EXPECT_EQ(Route.Allocations, 0u); + EXPECT_EQ(file.Closes, 0u); + entry.NullHandle = false; + + file.StatError = VFS::VFSError::IOError; + for (uint32_t boundary : {1u, 2u}) + { + file.Stats = 0; + file.StatFailureAt = boundary; + EXPECT_EQ(Capture().Error(), VFS::VFSError::IOError); + EXPECT_TRUE(Snapshot.Bytes().empty()); + EXPECT_FALSE(Route.InUse); + } + EXPECT_EQ(file.Closes, 2u); + EXPECT_EQ(Route.Allocations, Route.Frees); + file.StatError = VFS::VFSError::OK; + file.Stats = 0; + ASSERT_TRUE(Capture().Succeeded()); + ASSERT_EQ(Snapshot.Bytes().size(), 128u); + EXPECT_EQ(Snapshot.Bytes()[127], 127u); + Snapshot.Reset(); + EXPECT_EQ(Route.Allocations, Route.Frees); + } +} // namespace ZEngine::Cook diff --git a/ZEngine/tests/Cook/CookedPackageSnapshot_test.cpp b/ZEngine/tests/Cook/CookedPackageSnapshot_test.cpp new file mode 100644 index 000000000..13df4000d --- /dev/null +++ b/ZEngine/tests/Cook/CookedPackageSnapshot_test.cpp @@ -0,0 +1,1329 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "CookManifestGolden.h" +#include "CookTestAllocator.h" + +using ZEngine::Managers::CookedObjectSnapshot; +using ZEngine::Managers::CookedPackageSnapshot; + +namespace ZEngine::Cook +{ + namespace VFS = Core::VFS; + namespace Codec = Importers::AssetCodec; + using JSON = nlohmann::json; + + struct ClosureCacheDrain + { + Managers::CookedCpuClosureCache& Cache; + ~ClosureCacheDrain() + { + while (Cache.PumpOne()) + { + } + EXPECT_TRUE(Cache.Shutdown()); + } + }; + + class CookedPackageSnapshotTest : public testing::Test + { + protected: + Tests::FixedAllocator Allocator = {}; + Core::Memory::MemoryManager Memory = {}; + VFS::VFSMemoryBackend Backend = {}; + VFS::VFSContext Context = {}; + ECS::SceneComponentSchemaRegistry Schemas = {}; + ECS::SceneDiagnostics SceneDiagnostics = {}; + CookDiagnostics Diagnostics = {}; + CookLimits Limits = {.MaxWorkingBytes = ZKilo(256)}; + CookedPackageSnapshot Package = {}; + JSON Manifest = {}; + uuids::uuid SceneId = {}; + + void SetUp() override + { + Memory.Initialize(ZMega(8), {}); + Schemas.Initialize(&Memory.MainArena); + ASSERT_TRUE(ECS::RegisterBuiltInSceneComponentSchemas(Schemas, &SceneDiagnostics)); + ASSERT_TRUE(Schemas.FreezeForWorker(&SceneDiagnostics)); + ASSERT_TRUE(Backend.InitializeOwned(ZMega(2), 128).Succeeded()); + ASSERT_TRUE(Backend.CreateDir(VFS::VFSPath::Parse("/output").Value()).Succeeded()); + ASSERT_TRUE(Backend.CreateDir(VFS::VFSPath::Parse("/output/objects").Value()).Succeeded()); + Context.Initialize(&Memory.MainArena); + ASSERT_TRUE(Context.Mount(&Backend, VFS::VFSPath::Root(), 0).Succeeded()); + Manifest = JSON::parse(Tests::ManifestGolden.begin(), Tests::ManifestGolden.end()); + SceneId = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000001"); + const auto words = CookSchemaKeyWorkspaceWords(Schemas); + ASSERT_TRUE(words.Succeeded()); + std::vector workspace(words.Value()); + Digest256 key = {}; + ASSERT_TRUE(ComputeCookSchemaKey(Allocator, Schemas, {workspace.data(), workspace.size()}, &key).Succeeded()); + const auto hex = key.Hex(); + Manifest["schema_key"] = std::string(hex.data(), 64); + } + + void TearDown() override + { + Package.Reset(); + Context.Shutdown(); + EXPECT_EQ(Allocator.Statistics().LiveBytes, 0u); + } + + void WriteManifest(const JSON& value) + { + const auto text = value.dump(); + ASSERT_TRUE(Backend.WriteFile(VFS::VFSPath::Parse("/output/manifest.json").Value(), {reinterpret_cast(text.data()), text.size()}).Succeeded()); + } + + CookObjectDescriptor WriteObjectBytes(cstring extension, Core::Containers::ArrayView bytes) + { + Core::Hash::SHA256 hash = {}; + Digest256 digest = {}; + EXPECT_TRUE(hash.Update(bytes).Succeeded()); + EXPECT_TRUE(hash.Finalize(&digest).Succeeded()); + CookObjectDescriptor object = {}; + EXPECT_TRUE(MakeCookObjectDescriptor(digest, bytes.size(), extension, &object).Succeeded()); + const std::string path = std::string("/output/") + object.Path.CStr(); + EXPECT_TRUE(Backend.WriteFile(VFS::VFSPath::Parse(path.c_str()).Value(), bytes).Succeeded()); + return object; + } + + CookObjectDescriptor WriteObject(cstring extension, uint8_t value) + { + const std::array bytes = {value, 1, 2, 3, 4, 5, 6, 7}; + return WriteObjectBytes(extension, {bytes.data(), bytes.size()}); + } + + JSON ObjectJSON(const CookObjectDescriptor& object) + { + const auto hex = object.Digest.Hex(); + return { + {"path", object.Path.CStr()}, + {"digest", std::string(hex.data(), 64)}, + {"size", object.Size} + }; + } + + static std::array Pixel(uint8_t blue) + { + std::array bytes = {0x42, 0x4d, 58, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 40, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 24, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, blue, 20, 30, 0}; + return bytes; + } + + CookObjectDescriptor SetTexture(const uuids::uuid& texture, uint8_t blue) + { + const auto bytes = Pixel(blue); + const auto object = WriteObjectBytes("bmp", {bytes.data(), bytes.size()}); + Manifest["assets"] = JSON::array({ + {{"kind", "texture"}, {"uuid", uuids::to_string(texture)}, {"format", "image-bmp"}, {"adapter_revision", 1}, {"input_key", std::string(64, '1')}, {"object", ObjectJSON(object)}, {"dependencies", JSON::array()}} + }); + Manifest["scenes"][0]["roots"] = JSON::array({ + {{"kind", "texture"}, {"uuid", uuids::to_string(texture)}} + }); + WriteManifest(Manifest); + return object; + } + + VFS::VFSReadRootLease Root() + { + VFS::VFSReadRootLease root = {}; + EXPECT_TRUE(Context.AcquireReadRoot(VFS::VFSPath::Parse("/output").Value(), &root).Succeeded()); + return root; + } + + VFS::VFSResult Capture(VFS::VFSReadRootLease&& root, const uuids::uuid& scene) + { + return CookedPackageSnapshot::Capture(std::move(root), scene, Schemas, Limits, Allocator, &Diagnostics, &Package); + } + }; + + TEST_F(CookedPackageSnapshotTest, RetainsManifestAndRootAfterSourceManifestReplacement) + { + const auto scene_object = WriteObject(CookSceneObjectExtension, 17); + Manifest["scenes"][0]["object"] = ObjectJSON(scene_object); + WriteManifest(Manifest); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + EXPECT_FALSE(root.IsValid()); + ASSERT_TRUE(Package.IsValid()); + ASSERT_NE(Package.Scene(), nullptr); + EXPECT_EQ(Package.Scene()->Result.Object.Digest, scene_object.Digest); + const auto retained_digest = Package.ManifestDigest(); + WriteManifest(JSON::object()); + EXPECT_EQ(Package.ManifestDigest(), retained_digest); + CookCodecAllocationRoute route(Allocator); + CookedObjectSnapshot bytes = {}; + ASSERT_TRUE(Package.CaptureScene(route, 8, &bytes).Succeeded()); + ASSERT_EQ(bytes.Bytes().size(), 8u); + EXPECT_EQ(bytes.Bytes()[0], 17u); + bytes.Reset(); + CookedPackageSnapshot moved = std::move(Package); + EXPECT_FALSE(Package.IsValid()); + EXPECT_EQ(moved.Scene()->UUID, SceneId); + moved.Reset(); + } + + TEST_F(CookedPackageSnapshotTest, RejectsMissingSceneSchemaMismatchAndInvalidManifestWithoutConsumingRoot) + { + WriteManifest(Manifest); + auto root = Root(); + const auto absent = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000099"); + EXPECT_EQ(Capture(std::move(root), absent).Error(), VFS::VFSError::NotFound); + EXPECT_TRUE(root.IsValid()); + EXPECT_FALSE(Package.IsValid()); + auto changed = Manifest; + changed["schema_key"] = std::string(64, '4'); + WriteManifest(changed); + EXPECT_EQ(Capture(std::move(root), SceneId).Error(), VFS::VFSError::Unsupported); + EXPECT_TRUE(root.IsValid()); + changed = Manifest; + changed["recipe_revision"] = 0; + WriteManifest(changed); + EXPECT_TRUE(Capture(std::move(root), SceneId).Failed()); + EXPECT_TRUE(root.IsValid()); + EXPECT_FALSE(Package.IsValid()); + } + + TEST_F(CookedPackageSnapshotTest, PreflightSelectsOnlyTheRequestedSceneTransitiveClosure) + { + const auto mesh_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000004"); + const auto material_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000003"); + const auto texture_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000002"); + const auto environment_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000005"); + const auto identity = [](cstring kind, const uuids::uuid& uuid, cstring slot = nullptr) { + JSON result = { + {"kind", kind}, + {"uuid", uuids::to_string(uuid)} + }; + if (slot) + { + result["slot"] = slot; + } + return result; + }; + const auto asset = [&](cstring kind, const uuids::uuid& uuid, cstring format, cstring extension, uint8_t seed, JSON dependencies) { + return JSON{ + {"kind", kind}, + {"uuid", uuids::to_string(uuid)}, + {"format", format}, + {"adapter_revision", 1}, + {"input_key", std::string(64, '1')}, + {"object", ObjectJSON(WriteObject(extension, seed))}, + {"dependencies", dependencies} + }; + }; + Manifest["assets"] = JSON::array({ + asset("environment-map", environment_id, "zenvmap-v2-le64", "zenvmap", 1, JSON::array()), + asset("material", material_id, "zematerial-v1-json", "zematerial", 2, JSON::array({identity("texture", texture_id, "albedo")})), + asset("mesh", mesh_id, "zemesh-v1-le64", "zemesh", 3, JSON::array({identity("material", material_id, "submeshes/0/material")})), + asset("texture", texture_id, "image-png", "png", 4, JSON::array()), + }); + Manifest["scenes"][0]["roots"] = JSON::array({identity("mesh", mesh_id)}); + auto second = Manifest["scenes"][0]; + second["uuid"] = "00000000-0000-4000-8000-000000000009"; + second["source"] = "scenes/other.yaml"; + second["roots"] = JSON::array({identity("environment-map", environment_id)}); + Manifest["scenes"].push_back(second); + WriteManifest(Manifest); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + EXPECT_NE(Package.FindSelectedAsset({CookAssetKind::Mesh, mesh_id}), nullptr); + EXPECT_NE(Package.FindSelectedAsset({CookAssetKind::Material, material_id}), nullptr); + EXPECT_NE(Package.FindSelectedAsset({CookAssetKind::Texture, texture_id}), nullptr); + EXPECT_EQ(Package.FindSelectedAsset({CookAssetKind::EnvironmentMap, environment_id}), nullptr); + std::vector visited = {}; + const auto visit = [](void* context, const CookManifestAsset& entry) { + static_cast*>(context)->push_back(entry.Identity); + return VFS::VFSResult::Ok(); + }; + ASSERT_TRUE(Package.VisitSelectedAssets(visit, &visited).Succeeded()); + ASSERT_EQ(visited.size(), 3u); + EXPECT_EQ(visited[0], (CookAssetIdentity{CookAssetKind::Material, material_id})); + EXPECT_EQ(visited[1], (CookAssetIdentity{CookAssetKind::Mesh, mesh_id})); + EXPECT_EQ(visited[2], (CookAssetIdentity{CookAssetKind::Texture, texture_id})); + std::atomic cancelled = true; + EXPECT_EQ(Package.VisitSelectedAssets(visit, &visited, &cancelled).Error(), VFS::VFSError::Cancelled); + EXPECT_EQ(visited.size(), 3u); + EXPECT_EQ(Package.VisitSelectedAssets([](void*, const CookManifestAsset&) { return VFS::VFSResult::Fail(VFS::VFSError::PermissionDenied); }, nullptr).Error(), VFS::VFSError::PermissionDenied); + CookCodecAllocationRoute route(Allocator); + CookedObjectSnapshot bytes = {}; + EXPECT_EQ(Package.CaptureAsset({CookAssetKind::EnvironmentMap, environment_id}, route, 8, &bytes).Error(), VFS::VFSError::NotFound); + ASSERT_TRUE(Package.CaptureAsset({CookAssetKind::Texture, texture_id}, route, 8, &bytes).Succeeded()); + EXPECT_EQ(bytes.Bytes()[0], 4u); + } + + TEST_F(CookedPackageSnapshotTest, MeshReaderMatchesLexicographicSlotsAndRejectsMissingDependency) + { + const auto mesh_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000004"); + const auto material_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000003"); + Importers::AssetMesh mesh = {}; + mesh.MeshUUID = mesh_id; + mesh.Vertices.init(&Memory.MainArena, 24, 24); + mesh.Vertices[8] = 1.f; + mesh.Vertices[17] = 1.f; + mesh.Indices.init(&Memory.MainArena, 36); + mesh.SubMeshes.init(&Memory.MainArena, 12); + for (uint32_t i = 0; i < 12; ++i) + { + mesh.Indices.push(0); + mesh.Indices.push(1); + mesh.Indices.push(2); + mesh.SubMeshes.push({.MaterialUUID = material_id, .VertexCount = 3, .IndexCount = 3, .IndexOffset = i * 3, .IndexStreamOffset = i * 12, .VertexUnitStreamSize = 32, .IndexUnitStreamSize = 4, .TotalByteSize = 108}); + } + Importers::AssetNodeHierarchy hierarchy = {}; + hierarchy.NodeHierarchyUUID = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000005"); + hierarchy.MeshUUID = mesh_id; + hierarchy.Hierarchies.init(&Memory.MainArena, 1); + hierarchy.LocalTransforms.init(&Memory.MainArena, 1); + hierarchy.GlobalTransforms.init(&Memory.MainArena, 1); + hierarchy.Names.init(&Memory.MainArena, 1); + hierarchy.MaterialNames.init(&Memory.MainArena, 0); + hierarchy.NodeNames.init(&Memory.MainArena, 16); + hierarchy.NodeMeshes.init(&Memory.MainArena, 16); + hierarchy.NodeMaterials.init(&Memory.MainArena, 16); + ASSERT_EQ(Importers::AddNode(hierarchy, -1, 0), 0); + hierarchy.Names.push_use({}).init(&Memory.MainArena, "Root"); + hierarchy.NodeNames.insert(0, 0); + hierarchy.NodeMeshes.insert(0, 0); + Codec::ImportConfiguration config = {}; + config.VFS = &Context; + config.OutputAssetsPath.init(&Memory.MainArena, "/assets"); + config.OutputAssetFile.init(&Memory.MainArena, "mesh.zemesh"); + ASSERT_TRUE(Codec::SerializeMeshAssetFile(&Memory.MainArena, mesh, hierarchy, config).Succeeded()); + const auto opened = Context.Open(VFS::VFSPath::Parse("/assets/mesh.zemesh").Value(), VFS::VFSOpenFlags::Read); + ASSERT_TRUE(opened.Succeeded()); + const auto size = opened.Value()->Size(); + ASSERT_TRUE(size.Succeeded()); + std::vector bytes(static_cast(size.Value())); + const auto read = opened.Value()->ReadAll({bytes.data(), bytes.size()}); + ASSERT_TRUE(read.Succeeded()); + ASSERT_EQ(read.Value(), bytes.size()); + ASSERT_TRUE(Context.CloseChecked(opened.Value()).Succeeded()); + const auto mesh_object = WriteObjectBytes("zemesh", {bytes.data(), bytes.size()}); + const auto material_object = WriteObject("zematerial", 1); + std::vector slots = {}; + for (uint32_t i = 0; i < 12; ++i) + { + slots.push_back("submeshes/" + std::to_string(i) + "/material"); + } + std::sort(slots.begin(), slots.end()); + JSON dependencies = JSON::array(); + for (const auto& slot : slots) + { + dependencies.push_back({ + {"kind", "material"}, + {"uuid", uuids::to_string(material_id)}, + {"slot", slot} + }); + } + Manifest["assets"] = JSON::array({ + {{"kind", "material"}, {"uuid", uuids::to_string(material_id)}, {"format", "zematerial-v1-json"}, {"adapter_revision", 1}, {"input_key", std::string(64, '1')}, {"object", ObjectJSON(material_object)}, {"dependencies", JSON::array()}}, + { {"kind", "mesh"}, {"uuid", uuids::to_string(mesh_id)}, {"format", "zemesh-v1-le64"}, {"adapter_revision", 1}, {"input_key", std::string(64, '1')}, {"object", ObjectJSON(mesh_object)}, {"dependencies", dependencies}} + }); + Manifest["scenes"][0]["roots"] = JSON::array({ + {{"kind", "mesh"}, {"uuid", uuids::to_string(mesh_id)}} + }); + WriteManifest(Manifest); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + CookCodecAllocationRoute route(Allocator); + CookedObjectSnapshot object = {}; + const CookAssetIdentity identity = {CookAssetKind::Mesh, mesh_id}; + ASSERT_TRUE(Package.CaptureAsset(identity, route, ZKilo(64), &object).Succeeded()); + Core::Memory::ArenaAllocator arena = {}; + Memory.MainArena.CreateSubArena(ZKilo(128), &arena, "CookedMeshReaderTest"); + Importers::AssetMesh decoded = {}; + Importers::AssetNodeHierarchy decoded_hierarchy = {}; + ASSERT_TRUE(Managers::CookedAssetReader::ReadMesh(Package, identity, object, &arena, *route.WorkingMemoryBudget(), &decoded, &decoded_hierarchy).Succeeded()); + EXPECT_EQ(decoded.SubMeshes.size(), 12u); + size_t measured_bytes = 0; + ASSERT_TRUE(Codec::MeasureMeshAssetReadWorkspace(object.Source(), &measured_bytes, mesh_id).Succeeded()); + void* exact_memory = Memory.MainArena.AllocateNoZero(measured_bytes, DEFAULT_ALIGNMENT); + ASSERT_NE(exact_memory, nullptr); + Core::Memory::ArenaAllocator exact_arena = {}; + ASSERT_TRUE(exact_arena.InitializeBorrowed(exact_memory, measured_bytes)); + Importers::AssetMesh exact_mesh = {}; + Importers::AssetNodeHierarchy exact_hierarchy = {}; + const auto exact_result = Managers::CookedAssetReader::ReadMesh(Package, identity, object, &exact_arena, *route.WorkingMemoryBudget(), &exact_mesh, &exact_hierarchy); + ASSERT_TRUE(exact_result.Succeeded()) << "Measured workspace " << measured_bytes << ", error " << static_cast(exact_result.Error()); + const auto before = arena.m_current_offset; + struct RefusedBudget final : Core::Memory::IWorkingMemoryBudget + { + bool TryReserveWorkingBytes(size_t) noexcept override + { + return false; + } + void ReleaseWorkingBytes(size_t) noexcept override + { + ADD_FAILURE(); + } + } refused; + EXPECT_EQ(Managers::CookedAssetReader::ReadMesh(Package, identity, object, &arena, refused, &decoded, &decoded_hierarchy).Error(), VFS::VFSError::SizeLimitExceeded); + EXPECT_EQ(arena.m_current_offset, before); + Package.Reset(); + Manifest["assets"][1]["dependencies"].erase(2); // lexicographic slot 10 + WriteManifest(Manifest); + root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + EXPECT_EQ(Managers::CookedAssetReader::ReadMesh(Package, identity, object, &arena, *route.WorkingMemoryBudget(), &decoded, &decoded_hierarchy).Error(), VFS::VFSError::Corrupted); + EXPECT_EQ(arena.m_current_offset, before); + EXPECT_EQ(decoded.SubMeshes.size(), 12u); + + // The same mesh bytes may be reused across packages, but each package + // must still describe the material edge encoded by every submesh. + const auto other_material = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000006"); + const auto make_material = [&](const uuids::uuid& id, float opacity) { + Codec::MaterialAssetDocument value = {}; + value.UUID = id; + value.Name[0] = 'M'; + value.Factors[0] = opacity; + Core::Containers::FixedArray paths = {}; + std::string text = {}; + const auto write = [](void* context, Core::Containers::ArrayView input) { + static_cast(context)->append(reinterpret_cast(input.data()), input.size()); + return VFS::VFSResult::Ok(); + }; + EXPECT_TRUE(Codec::WriteCanonicalMaterialAsset(value, paths, write, &text).Succeeded()); + return WriteObjectBytes("zematerial", {reinterpret_cast(text.data()), text.size()}); + }; + const auto first_material_object = make_material(material_id, 1.f); + const auto other_material_object = make_material(other_material, 1.f); + const auto material_entry = [&](const uuids::uuid& id, const CookObjectDescriptor& descriptor) { + return JSON{ + {"kind", "material"}, + {"uuid", uuids::to_string(id)}, + {"format", "zematerial-v1-json"}, + {"adapter_revision", 1}, + {"input_key", std::string(64, '1')}, + {"object", ObjectJSON(descriptor)}, + {"dependencies", JSON::array()} + }; + }; + Manifest["assets"] = JSON::array({ + material_entry(material_id, first_material_object), material_entry(other_material, other_material_object), {{"kind", "mesh"}, {"uuid", uuids::to_string(mesh_id)}, {"format", "zemesh-v1-le64"}, {"adapter_revision", 1}, {"input_key", std::string(64, '1')}, {"object", ObjectJSON(mesh_object)}, {"dependencies", dependencies}} + }); + Manifest["scenes"][0]["roots"] = JSON::array({ + {{"kind", "material"}, {"uuid", uuids::to_string(other_material)}}, + { {"kind", "mesh"}, {"uuid", uuids::to_string(mesh_id)}} + }); + WriteManifest(Manifest); + Package.Reset(); + root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + Core::Memory::SlabAllocationRoute cache_route = {}; + ASSERT_TRUE(cache_route.Initialize(&Memory.MainArena, ZMega(1))); + const Managers::CookedCpuAssetLimits cache_limits = {.MaxAssets = 3, .MaxObjectBytes = ZKilo(64), .MaxMeshWorkspaceBytes = ZKilo(128)}; + Managers::CookedCpuClosureCache cache = {}; + ASSERT_TRUE(cache.Initialize(cache_route, Schemas, cache_limits, 3, 2, 4)); + ClosureCacheDrain drain{cache}; + Managers::CookedCpuClosureCache::Lease valid = {}, compatible = {}, invalid = {}; + ASSERT_TRUE(cache.Request(std::move(Package), &valid)); + ASSERT_TRUE(cache.PumpOne()); + Managers::CookedCpuClosureCache::Result result = {}; + ASSERT_TRUE(valid.Poll(&result)); + ASSERT_EQ(result.StateValue, Managers::CookedCpuClosureCache::State::Ready) << "VFS error " << static_cast(result.Error) << ", workspace " << measured_bytes << ", route peak " << cache_route.PeakBytes() << ", capacity " << cache_route.Capacity(); + const auto* retained_mesh = result.Assets->FindMesh(identity); + ASSERT_NE(retained_mesh, nullptr); + const CookAssetIdentity material_identity = {CookAssetKind::Material, material_id}; + const auto* retained_material = result.Assets->FindMaterial(material_identity); + ASSERT_NE(retained_material, nullptr); + const auto replacement_material = make_material(material_id, .5f); + Manifest["assets"][0]["object"] = ObjectJSON(replacement_material); + WriteManifest(Manifest); + CookedPackageSnapshot changed_material = {}; + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), SceneId, Schemas, Limits, Allocator, &Diagnostics, &changed_material).Succeeded()); + ASSERT_TRUE(cache.Request(std::move(changed_material), &compatible)); + ASSERT_TRUE(cache.PumpOne()); + Managers::CookedCpuClosureCache::Result compatible_result = {}; + ASSERT_TRUE(compatible.Poll(&compatible_result)); + ASSERT_EQ(compatible_result.StateValue, Managers::CookedCpuClosureCache::State::Ready); + EXPECT_EQ(compatible_result.Assets->FindMesh(identity), retained_mesh); + ASSERT_NE(compatible_result.Assets->FindMaterial(material_identity), nullptr); + EXPECT_NE(compatible_result.Assets->FindMaterial(material_identity), retained_material); + EXPECT_FLOAT_EQ(retained_material->Factors[0], 1.f); + EXPECT_FLOAT_EQ(compatible_result.Assets->FindMaterial(material_identity)->Factors[0], .5f); + const auto retained_bytes = cache_route.AllocatedBytes(); + Manifest["assets"][2]["dependencies"][2]["uuid"] = uuids::to_string(other_material); + WriteManifest(Manifest); + CookedPackageSnapshot inconsistent = {}; + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), SceneId, Schemas, Limits, Allocator, &Diagnostics, &inconsistent).Succeeded()); + ASSERT_TRUE(cache.Request(std::move(inconsistent), &invalid)); + ASSERT_TRUE(cache.PumpOne()); + Managers::CookedCpuClosureCache::Result failed = {}; + ASSERT_TRUE(invalid.Poll(&failed)); + EXPECT_EQ(failed.StateValue, Managers::CookedCpuClosureCache::State::Failed); + EXPECT_EQ(failed.Error, VFS::VFSError::Corrupted); + EXPECT_EQ(cache_route.AllocatedBytes(), retained_bytes); + ASSERT_TRUE(valid.Poll(&result)); + EXPECT_EQ(result.Assets->FindMesh(identity), retained_mesh); + EXPECT_EQ(retained_mesh->SubMeshes[10].MaterialUUID, material_id); + valid.Reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(compatible.Poll(&compatible_result)); + EXPECT_EQ(compatible_result.Assets->FindMesh(identity), retained_mesh); + compatible.Reset(); + invalid.Reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.Shutdown()); + } + + TEST_F(CookedPackageSnapshotTest, IndependentCpuVersionsSurvivePackageReplacementAndCorruption) + { + const auto texture_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000002"); + const CookAssetIdentity identity = {CookAssetKind::Texture, texture_id}; + auto bytes = Pixel(10); + (void) SetTexture(texture_id, 10); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + CookCodecAllocationRoute route(Allocator); + Managers::CookedCpuAssetLimits limits = {.MaxAssets = 1, .MaxObjectBytes = 58, .MaxMeshWorkspaceBytes = ZKilo(64)}; + Managers::CookedCpuAssetSet old_assets = {}; + ASSERT_TRUE(Managers::CookedCpuAssetSet::Capture(Package, route, limits, &old_assets).Succeeded()); + ASSERT_NE(old_assets.FindTexture(identity), nullptr); + EXPECT_EQ(old_assets.FindTexture(identity)->Pixels()[2], 10u); + + bytes[54] = 90; + const auto second = WriteObjectBytes("bmp", {bytes.data(), bytes.size()}); + Manifest["assets"][0]["object"] = ObjectJSON(second); + Manifest["assets"][0]["input_key"] = std::string(64, '2'); + WriteManifest(Manifest); + CookedPackageSnapshot next_package = {}; + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), SceneId, Schemas, Limits, Allocator, &Diagnostics, &next_package).Succeeded()); + Managers::CookedCpuAssetSet next_assets = {}; + ASSERT_TRUE(Managers::CookedCpuAssetSet::Capture(next_package, route, limits, &next_assets).Succeeded()); + EXPECT_NE(old_assets.FindVersion(identity)->ObjectDigest, next_assets.FindVersion(identity)->ObjectDigest); + EXPECT_EQ(old_assets.FindTexture(identity)->Pixels()[2], 10u); + EXPECT_EQ(next_assets.FindTexture(identity)->Pixels()[2], 90u); + bytes[54] = 42; + const std::string path = std::string("/output/") + second.Path.CStr(); + ASSERT_TRUE(Backend.WriteFile(VFS::VFSPath::Parse(path.c_str()).Value(), {bytes.data(), bytes.size()}).Succeeded()); + Managers::CookedCpuAssetSet rejected = {}; + const auto baseline = Allocator.Statistics().LiveBytes; + EXPECT_EQ(Managers::CookedCpuAssetSet::Capture(next_package, route, limits, &rejected).Error(), VFS::VFSError::Corrupted); + EXPECT_FALSE(rejected.IsValid()); + EXPECT_EQ(Allocator.Statistics().LiveBytes, baseline); + Package.Reset(); + next_package.Reset(); + EXPECT_EQ(old_assets.FindTexture(identity)->Pixels()[2], 10u); + EXPECT_EQ(next_assets.FindTexture(identity)->Pixels()[2], 90u); + } + + TEST_F(CookedPackageSnapshotTest, EmptyClosureHasAValidOwnerWithoutAssetAllocations) + { + WriteManifest(Manifest); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + CookCodecAllocationRoute route(Allocator); + const auto baseline = Allocator.Statistics().LiveBytes; + Managers::CookedCpuAssetLimits limits = {.MaxAssets = 0, .MaxObjectBytes = 1}; + Managers::CookedCpuAssetSet assets = {}; + ASSERT_TRUE(Managers::CookedCpuAssetSet::Capture(Package, route, limits, &assets).Succeeded()); + EXPECT_TRUE(assets.IsValid()); + EXPECT_EQ(assets.Size(), 0u); + EXPECT_EQ(assets.VersionAt(0), nullptr); + EXPECT_EQ(Allocator.Statistics().LiveBytes, baseline); + EXPECT_EQ(Managers::CookedCpuAssetSet::Capture(Package, route, limits, &assets).Error(), VFS::VFSError::InvalidPath); + EXPECT_TRUE(assets.IsValid()); + } + + TEST_F(CookedPackageSnapshotTest, ClosureCacheSharesOneBuildAndRetiresOnlyOnWorker) + { + using Cache = Managers::CookedCpuClosureCache; + const auto texture = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000002"); + const CookAssetIdentity identity = {CookAssetKind::Texture, texture}; + (void) SetTexture(texture, 10); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + CookedPackageSnapshot other = {}; + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), SceneId, Schemas, Limits, Allocator, &Diagnostics, &other).Succeeded()); + Core::Memory::SlabAllocationRoute route = {}; + ASSERT_TRUE(route.Initialize(&Memory.MainArena, ZMega(1))); + const Managers::CookedCpuAssetLimits limits = {.MaxAssets = 1, .MaxObjectBytes = 58, .MaxMeshWorkspaceBytes = ZKilo(64)}; + Cache cache = {}; + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 2, 2, 8)); + ClosureCacheDrain drain{cache}; + const auto empty_bytes = route.AllocatedBytes(); + Cache::Lease first = {}, second = {}, refused = {}; + ASSERT_TRUE(cache.Request(std::move(Package), &first)); + EXPECT_FALSE(Package.IsValid()); + EXPECT_FALSE(first.Retain(nullptr)); + ASSERT_TRUE(first.Retain(&second)); + EXPECT_FALSE(first.Retain(&second)); + EXPECT_FALSE(first.Retain(&refused)); + EXPECT_TRUE(other.IsValid()); + EXPECT_FALSE(cache.Request(std::move(other), &refused)); + EXPECT_FALSE(refused); + EXPECT_TRUE(other.IsValid()); + Cache::Result first_result = {}, second_result = {}; + ASSERT_TRUE(first.Poll(&first_result)); + EXPECT_EQ(first_result.StateValue, Cache::State::Pending); + EXPECT_FALSE(cache.Shutdown()); + ASSERT_TRUE(cache.PumpOne()); + EXPECT_FALSE(cache.PumpOne()); + ASSERT_TRUE(first.Poll(&first_result)); + ASSERT_TRUE(second.Poll(&second_result)); + ASSERT_EQ(first_result.StateValue, Cache::State::Ready); + EXPECT_EQ(first_result.Assets, second_result.Assets); + ASSERT_NE(first_result.Assets->FindTexture(identity), nullptr); + EXPECT_EQ(first_result.Assets->FindTexture(identity)->Pixels()[2], 10u); + const auto live_bytes = route.AllocatedBytes(); + EXPECT_GT(live_bytes, empty_bytes); + first.Reset(); + EXPECT_EQ(route.AllocatedBytes(), live_bytes); + ASSERT_TRUE(second.Poll(&second_result)); + EXPECT_EQ(second_result.Assets->FindTexture(identity)->Pixels()[2], 10u); + ASSERT_TRUE(second.Retain(&first)); + second.Reset(); + ASSERT_TRUE(first.Poll(&first_result)); + EXPECT_EQ(first_result.Assets, second_result.Assets); + first.Reset(); + EXPECT_EQ(route.AllocatedBytes(), live_bytes); + ASSERT_TRUE(cache.PumpOne()); + EXPECT_EQ(route.AllocatedBytes(), empty_bytes); + ASSERT_TRUE(cache.Request(std::move(other), &refused)); + EXPECT_FALSE(other.IsValid()); + refused.Reset(); + ASSERT_TRUE(cache.PumpOne()); + EXPECT_EQ(route.AllocatedBytes(), empty_bytes); + ASSERT_TRUE(cache.Shutdown()); + EXPECT_EQ(route.AllocatedBytes(), 0u); + } + + TEST_F(CookedPackageSnapshotTest, ClosureCacheCancellationKeepsOtherWaitersAndDrainsLastCancel) + { + using Cache = Managers::CookedCpuClosureCache; + const auto texture = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000002"); + (void) SetTexture(texture, 10); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + CookedPackageSnapshot other = {}; + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), SceneId, Schemas, Limits, Allocator, &Diagnostics, &other).Succeeded()); + Core::Memory::SlabAllocationRoute base = {}; + ASSERT_TRUE(base.Initialize(&Memory.MainArena, ZMega(1))); + struct BlockingRoute final : Core::Memory::IAllocationRoute + { + Core::Memory::SlabAllocationRoute* Base = nullptr; + std::atomic Armed = false; + std::atomic Entered = false; + std::atomic Resume = false; + void* Allocate(size_t size) noexcept override + { + void* memory = Base->Allocate(size); + if (Armed.exchange(false, std::memory_order_acq_rel)) + { + Entered.store(true, std::memory_order_release); + while (!Resume.load(std::memory_order_acquire)) + { + std::this_thread::yield(); + } + } + return memory; + } + void* Reallocate(void* pointer, size_t old_size, size_t new_size) noexcept override + { + return Base->Reallocate(pointer, old_size, new_size); + } + void Free(void* pointer) noexcept override + { + Base->Free(pointer); + } + Core::Memory::IWorkingMemoryBudget* WorkingMemoryBudget() noexcept override + { + return Base->WorkingMemoryBudget(); + } + } route; + route.Base = &base; + const Managers::CookedCpuAssetLimits limits = {.MaxAssets = 1, .MaxObjectBytes = 58, .MaxMeshWorkspaceBytes = ZKilo(64)}; + Cache cache = {}; + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 1, 2, 8)); + ClosureCacheDrain drain{cache}; + const auto baseline = base.AllocatedBytes(); + Cache::Lease first = {}, second = {}; + ASSERT_TRUE(cache.Request(std::move(Package), &first)); + route.Armed.store(true, std::memory_order_release); + std::atomic pumped = false; + std::thread worker([&] { pumped.store(cache.PumpOne(), std::memory_order_release); }); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!route.Entered.load(std::memory_order_acquire) && std::chrono::steady_clock::now() < deadline) + { + std::this_thread::yield(); + } + const bool entered = route.Entered.load(std::memory_order_acquire); + const bool joined = entered && cache.Request(std::move(other), &second); + Cache::Result pending = {}; + const bool pending_visible = joined && second.Poll(&pending) && pending.StateValue == Cache::State::Pending; + const bool shutdown_while_busy = cache.Shutdown(); + first.Reset(); + route.Resume.store(true, std::memory_order_release); + worker.join(); + ASSERT_TRUE(entered); + ASSERT_TRUE(joined); + EXPECT_TRUE(pending_visible); + EXPECT_FALSE(shutdown_while_busy); + ASSERT_TRUE(pumped.load(std::memory_order_acquire)); + Cache::Result result = {}; + ASSERT_TRUE(second.Poll(&result)); + EXPECT_EQ(result.StateValue, Cache::State::Ready); + EXPECT_TRUE(other.IsValid()); + second.Reset(); + ASSERT_TRUE(cache.PumpOne()); + EXPECT_EQ(base.AllocatedBytes(), baseline); + + Cache::Lease last = {}; + ASSERT_TRUE(cache.Request(std::move(other), &last)); + route.Entered.store(false, std::memory_order_release); + route.Resume.store(false, std::memory_order_release); + route.Armed.store(true, std::memory_order_release); + pumped.store(false, std::memory_order_release); + std::thread cancelled_worker([&] { pumped.store(cache.PumpOne(), std::memory_order_release); }); + const auto cancel_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!route.Entered.load(std::memory_order_acquire) && std::chrono::steady_clock::now() < cancel_deadline) + { + std::this_thread::yield(); + } + const bool cancelling = route.Entered.load(std::memory_order_acquire); + last.Reset(); + route.Resume.store(true, std::memory_order_release); + cancelled_worker.join(); + ASSERT_TRUE(cancelling); + ASSERT_TRUE(pumped.load(std::memory_order_acquire)); + EXPECT_EQ(base.AllocatedBytes(), baseline); + EXPECT_FALSE(cache.PumpOne()); + ASSERT_TRUE(cache.Shutdown()); + } + + TEST_F(CookedPackageSnapshotTest, ClosureCacheAdmitsNewWorkWhileRetirementWorkerIsBlocked) + { + using Cache = Managers::CookedCpuClosureCache; + const auto texture = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000002"); + (void) SetTexture(texture, 10); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + CookedPackageSnapshot next = {}; + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), SceneId, Schemas, Limits, Allocator, &Diagnostics, &next).Succeeded()); + Core::Memory::SlabAllocationRoute base = {}; + ASSERT_TRUE(base.Initialize(&Memory.MainArena, ZMega(1))); + struct BlockingFreeRoute final : Core::Memory::IAllocationRoute + { + Core::Memory::SlabAllocationRoute* Base = nullptr; + std::atomic Armed = false, Entered = false, Resume = false; + void* Allocate(size_t size) noexcept override + { + return Base->Allocate(size); + } + void* Reallocate(void* pointer, size_t old_size, size_t new_size) noexcept override + { + return Base->Reallocate(pointer, old_size, new_size); + } + void Free(void* pointer) noexcept override + { + if (Armed.exchange(false, std::memory_order_acq_rel)) + { + Entered.store(true, std::memory_order_release); + while (!Resume.load(std::memory_order_acquire)) + std::this_thread::yield(); + } + Base->Free(pointer); + } + Core::Memory::IWorkingMemoryBudget* WorkingMemoryBudget() noexcept override + { + return Base->WorkingMemoryBudget(); + } + } route; + route.Base = &base; + const Managers::CookedCpuAssetLimits limits = {.MaxAssets = 1, .MaxObjectBytes = 58, .MaxMeshWorkspaceBytes = ZKilo(64)}; + Cache cache = {}; + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 2, 2, 8)); + ClosureCacheDrain drain{cache}; + Cache::Lease first = {}, second = {}; + ASSERT_TRUE(cache.Request(std::move(Package), &first)); + ASSERT_TRUE(cache.PumpOne()); + first.Reset(); + Cache::Statistics before = {}; + ASSERT_TRUE(cache.TryGetStatistics(&before)); + EXPECT_EQ(before.Retiring, 1u); + EXPECT_EQ(before.Leases, 0u); + route.Armed.store(true, std::memory_order_release); + std::atomic pumped = false; + std::thread worker([&] { pumped.store(cache.PumpOne(), std::memory_order_release); }); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!route.Entered.load(std::memory_order_acquire) && std::chrono::steady_clock::now() < deadline) + std::this_thread::yield(); + const bool entered = route.Entered.load(std::memory_order_acquire); + const bool admitted = entered && cache.Request(std::move(next), &second); + Cache::Statistics busy = {.Closures = 99}; + const bool snapshot = cache.TryGetStatistics(&busy); + const bool second_pump = cache.PumpOne(); + route.Resume.store(true, std::memory_order_release); + worker.join(); + ASSERT_TRUE(entered); + ASSERT_TRUE(admitted); + EXPECT_FALSE(snapshot); + EXPECT_EQ(busy.Closures, 99u); + EXPECT_FALSE(second_pump); + ASSERT_TRUE(pumped.load(std::memory_order_acquire)); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result result = {}; + ASSERT_TRUE(second.Poll(&result)); + EXPECT_EQ(result.StateValue, Cache::State::Ready); + second.Reset(); + ASSERT_TRUE(cache.PumpOne()); + Cache::Statistics final = {}; + ASSERT_TRUE(cache.TryGetStatistics(&final)); + EXPECT_EQ(final.Closures, 0u); + EXPECT_EQ(final.Leases, 0u); + EXPECT_EQ(final.AssetVersions, 0u); + } + + TEST_F(CookedPackageSnapshotTest, ClosureCacheKeepsFailureIsolatedAndAllowsRetry) + { + using Cache = Managers::CookedCpuClosureCache; + const auto texture = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000002"); + const auto object = SetTexture(texture, 10); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + CookedPackageSnapshot retry_package = {}; + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), SceneId, Schemas, Limits, Allocator, &Diagnostics, &retry_package).Succeeded()); + Core::Memory::SlabAllocationRoute route = {}; + ASSERT_TRUE(route.Initialize(&Memory.MainArena, ZMega(1))); + const Managers::CookedCpuAssetLimits limits = {.MaxAssets = 1, .MaxObjectBytes = 58, .MaxMeshWorkspaceBytes = ZKilo(64)}; + Cache cache = {}; + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 2, 2, 8)); + ClosureCacheDrain drain{cache}; + Cache::Lease failed = {}, retry = {}; + ASSERT_TRUE(cache.Request(std::move(Package), &failed)); + const std::string path = std::string("/output/") + object.Path.CStr(); + auto bytes = Pixel(42); + ASSERT_TRUE(Backend.WriteFile(VFS::VFSPath::Parse(path.c_str()).Value(), {bytes.data(), bytes.size()}).Succeeded()); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result failed_result = {}; + ASSERT_TRUE(failed.Poll(&failed_result)); + EXPECT_EQ(failed_result.StateValue, Cache::State::Failed); + EXPECT_EQ(failed_result.Error, VFS::VFSError::Corrupted); + EXPECT_EQ(failed_result.Assets, nullptr); + bytes = Pixel(10); + ASSERT_TRUE(Backend.WriteFile(VFS::VFSPath::Parse(path.c_str()).Value(), {bytes.data(), bytes.size()}).Succeeded()); + ASSERT_TRUE(cache.Request(std::move(retry_package), &retry)); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result retry_result = {}; + ASSERT_TRUE(retry.Poll(&retry_result)); + EXPECT_EQ(retry_result.StateValue, Cache::State::Ready); + ASSERT_TRUE(failed.Poll(&failed_result)); + EXPECT_EQ(failed_result.StateValue, Cache::State::Failed); + failed.Reset(); + retry.Reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.PumpOne()); + EXPECT_FALSE(cache.PumpOne()); + ASSERT_TRUE(cache.Shutdown()); + } + + TEST_F(CookedPackageSnapshotTest, ClosureCacheKeepsOldAndNewVersionsIndependent) + { + using Cache = Managers::CookedCpuClosureCache; + const auto texture = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000002"); + const CookAssetIdentity identity = {CookAssetKind::Texture, texture}; + (void) SetTexture(texture, 10); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + Core::Memory::SlabAllocationRoute route = {}; + ASSERT_TRUE(route.Initialize(&Memory.MainArena, ZMega(1))); + const Managers::CookedCpuAssetLimits limits = {.MaxAssets = 1, .MaxObjectBytes = 58, .MaxMeshWorkspaceBytes = ZKilo(64)}; + Cache cache = {}; + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 2, 2, 8)); + ClosureCacheDrain drain{cache}; + Cache::Lease old_lease = {}, new_lease = {}; + ASSERT_TRUE(cache.Request(std::move(Package), &old_lease)); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result old_result = {}; + ASSERT_TRUE(old_lease.Poll(&old_result)); + ASSERT_EQ(old_result.StateValue, Cache::State::Ready); + ASSERT_NE(old_result.Assets->FindTexture(identity), nullptr); + EXPECT_EQ(old_result.Assets->FindTexture(identity)->Pixels()[2], 10u); + + (void) SetTexture(texture, 90); + CookedPackageSnapshot replacement = {}; + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), SceneId, Schemas, Limits, Allocator, &Diagnostics, &replacement).Succeeded()); + ASSERT_TRUE(cache.Request(std::move(replacement), &new_lease)); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result new_result = {}; + ASSERT_TRUE(new_lease.Poll(&new_result)); + ASSERT_EQ(new_result.StateValue, Cache::State::Ready); + ASSERT_NE(new_result.Assets->FindTexture(identity), nullptr); + EXPECT_NE(old_result.Assets, new_result.Assets); + EXPECT_NE(old_result.Assets->FindVersion(identity)->ObjectDigest, new_result.Assets->FindVersion(identity)->ObjectDigest); + EXPECT_EQ(old_result.Assets->FindTexture(identity)->Pixels()[2], 10u); + EXPECT_EQ(new_result.Assets->FindTexture(identity)->Pixels()[2], 90u); + old_lease.Reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(new_lease.Poll(&new_result)); + EXPECT_EQ(new_result.Assets->FindTexture(identity)->Pixels()[2], 90u); + new_lease.Reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.Shutdown()); + } + + TEST_F(CookedPackageSnapshotTest, CookedTextureUploadRetainsExactSourceAndPreservesLeaseOnRefusal) + { + using Cache = Managers::CookedCpuClosureCache; + using Upload = Rendering::Textures::CookedTextureUpload; + const auto texture = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000002"); + const CookAssetIdentity identity = {CookAssetKind::Texture, texture}; + const auto object = SetTexture(texture, 10); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + Core::Memory::SlabAllocationRoute route = {}; + ASSERT_TRUE(route.Initialize(&Memory.MainArena, ZMega(1))); + const Managers::CookedCpuAssetLimits limits = {.MaxAssets = 1, .MaxObjectBytes = 58, .MaxMeshWorkspaceBytes = ZKilo(64)}; + Cache cache = {}; + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 1, 2, 4)); + ClosureCacheDrain drain{cache}; + Cache::Lease lease = {}; + ASSERT_TRUE(cache.Request(std::move(Package), &lease)); + Upload upload = {}; + EXPECT_EQ(upload.Poll(), Upload::Readiness::Failed); + EXPECT_FALSE(upload.Capture(std::move(lease), identity)); // Still queued. + EXPECT_TRUE(lease); + ASSERT_TRUE(cache.PumpOne()); + EXPECT_FALSE(upload.Capture(std::move(lease), {CookAssetKind::Mesh, texture})); + EXPECT_TRUE(lease); + ASSERT_TRUE(upload.Capture(std::move(lease), identity)); + EXPECT_FALSE(lease); + EXPECT_EQ(upload.State(), Upload::Phase::SourceReady); + EXPECT_EQ(upload.Poll(), Upload::Readiness::Pending); + EXPECT_EQ(upload.Identity(), identity); + EXPECT_EQ(upload.Digest(), object.Digest); + EXPECT_EQ(upload.SubmittedBinding(), nullptr); + EXPECT_FALSE(cache.Shutdown()); // Candidate owns the only closure lease. + auto no_device = std::make_unique(); + EXPECT_FALSE(upload.Prepare(*no_device)); + EXPECT_EQ(upload.State(), Upload::Phase::SourceReady); + EXPECT_FALSE(upload.StageOne()); + EXPECT_FALSE(upload.Submit(0, 0, {})); + upload.Reset(); + upload.Reset(); + EXPECT_EQ(upload.State(), Upload::Phase::Empty); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.Shutdown()); + } + + TEST_F(CookedPackageSnapshotTest, ClosureCacheRequiresItsFrozenSchemaRegistryAndBoundedCapacity) + { + using Cache = Managers::CookedCpuClosureCache; + WriteManifest(Manifest); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + ECS::SceneComponentSchemaRegistry other_schemas = {}; + other_schemas.Initialize(&Memory.MainArena); + ASSERT_TRUE(ECS::RegisterBuiltInSceneComponentSchemas(other_schemas, &SceneDiagnostics)); + ASSERT_TRUE(other_schemas.FreezeForWorker(&SceneDiagnostics)); + CookedPackageSnapshot other = {}; + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), SceneId, other_schemas, Limits, Allocator, &Diagnostics, &other).Succeeded()); + Core::Memory::SlabAllocationRoute route = {}; + ASSERT_TRUE(route.Initialize(&Memory.MainArena, ZMega(1))); + const Managers::CookedCpuAssetLimits limits = {.MaxAssets = 0, .MaxObjectBytes = 1}; + Cache cache = {}; + EXPECT_FALSE(cache.Initialize(route, Schemas, limits, Cache::MaxClosures + 1, 1, 0)); + EXPECT_EQ(route.AllocatedBytes(), 0u); + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 1, 1, 0)); + ClosureCacheDrain drain{cache}; + Cache::Lease lease = {}; + EXPECT_FALSE(cache.Request(std::move(other), &lease)); + EXPECT_TRUE(other.IsValid()); + EXPECT_FALSE(lease); + ASSERT_TRUE(cache.Request(std::move(Package), &lease)); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result result = {}; + ASSERT_TRUE(lease.Poll(&result)); + ASSERT_EQ(result.StateValue, Cache::State::Ready); + EXPECT_EQ(result.Assets->Size(), 0u); + lease.Reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.Shutdown()); + } + + TEST_F(CookedPackageSnapshotTest, DifferentScenesShareAnExactAssetVersionAndRespectVersionCapacity) + { + using Cache = Managers::CookedCpuClosureCache; + const auto texture = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000002"); + const auto other_scene = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000009"); + const CookAssetIdentity identity = {CookAssetKind::Texture, texture}; + const auto object = SetTexture(texture, 10); + auto scene = Manifest["scenes"][0]; + scene["uuid"] = uuids::to_string(other_scene); + scene["source"] = "scenes/other.yaml"; + Manifest["scenes"].push_back(scene); + WriteManifest(Manifest); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + CookedPackageSnapshot other = {}; + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), other_scene, Schemas, Limits, Allocator, &Diagnostics, &other).Succeeded()); + Core::Memory::SlabAllocationRoute route = {}; + ASSERT_TRUE(route.Initialize(&Memory.MainArena, ZMega(1))); + const Managers::CookedCpuAssetLimits limits = {.MaxAssets = 1, .MaxObjectBytes = 58, .MaxMeshWorkspaceBytes = ZKilo(64)}; + Cache cache = {}; + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 3, 2, 1)); + ClosureCacheDrain drain{cache}; + const auto baseline = route.AllocatedBytes(); + Cache::Lease first = {}, second = {}, exhausted = {}, retry = {}; + ASSERT_TRUE(cache.Request(std::move(Package), &first)); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result first_result = {}; + ASSERT_TRUE(first.Poll(&first_result)); + ASSERT_EQ(first_result.StateValue, Cache::State::Ready); + const auto* original = first_result.Assets->FindTexture(identity); + ASSERT_NE(original, nullptr); + const std::string path = std::string("/output/") + object.Path.CStr(); + ASSERT_TRUE(Backend.Remove(VFS::VFSPath::Parse(path.c_str()).Value()).Succeeded()); + ASSERT_TRUE(cache.Request(std::move(other), &second)); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result second_result = {}; + ASSERT_TRUE(second.Poll(&second_result)); + ASSERT_EQ(second_result.StateValue, Cache::State::Ready); + EXPECT_NE(first_result.Assets, second_result.Assets); + EXPECT_EQ(original, second_result.Assets->FindTexture(identity)); + EXPECT_EQ(second_result.Assets->FindTexture(identity)->Pixels()[2], 10u); + + (void) SetTexture(texture, 90); + CookedPackageSnapshot replacement = {}; + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), SceneId, Schemas, Limits, Allocator, &Diagnostics, &replacement).Succeeded()); + ASSERT_TRUE(cache.Request(std::move(replacement), &exhausted)); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result failure = {}; + ASSERT_TRUE(exhausted.Poll(&failure)); + EXPECT_EQ(failure.StateValue, Cache::State::Failed); + EXPECT_EQ(failure.Error, VFS::VFSError::SizeLimitExceeded); + first.Reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(second.Poll(&second_result)); + EXPECT_EQ(second_result.Assets->FindTexture(identity), original); + second.Reset(); + ASSERT_TRUE(cache.PumpOne()); + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), SceneId, Schemas, Limits, Allocator, &Diagnostics, &replacement).Succeeded()); + ASSERT_TRUE(cache.Request(std::move(replacement), &retry)); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result retry_result = {}; + ASSERT_TRUE(retry.Poll(&retry_result)); + ASSERT_EQ(retry_result.StateValue, Cache::State::Ready); + ASSERT_NE(retry_result.Assets->FindTexture(identity), nullptr); + EXPECT_EQ(retry_result.Assets->FindTexture(identity)->Pixels()[2], 90u); + exhausted.Reset(); + retry.Reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.PumpOne()); + EXPECT_EQ(route.AllocatedBytes(), baseline); + ASSERT_TRUE(cache.Shutdown()); + } + + TEST_F(CookedPackageSnapshotTest, SharedMaterialStillRejectsDifferentManifestDependencies) + { + using Cache = Managers::CookedCpuClosureCache; + const auto material_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000003"); + const auto first_texture = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000004"); + const auto other_texture = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000005"); + const CookAssetIdentity material_key = {CookAssetKind::Material, material_id}; + auto pixel = Pixel(10); + const auto first_object = WriteObjectBytes("bmp", {pixel.data(), pixel.size()}); + pixel = Pixel(90); + const auto other_object = WriteObjectBytes("bmp", {pixel.data(), pixel.size()}); + Codec::MaterialAssetDocument material = {}; + material.UUID = material_id; + material.Name[0] = 'M'; + material.Textures[0].UUID = first_texture; + ASSERT_EQ(Helpers::secure_strcpy(material.Textures[0].Path, sizeof(material.Textures[0].Path), first_object.Path.CStr()), Helpers::MEMORY_OP_SUCCESS); + Core::Containers::FixedArray paths = {}; + paths[0] = first_object.Path; + std::string material_bytes = {}; + const auto write = [](void* context, Core::Containers::ArrayView bytes) { + static_cast(context)->append(reinterpret_cast(bytes.data()), bytes.size()); + return VFS::VFSResult::Ok(); + }; + ASSERT_TRUE(Codec::WriteCanonicalMaterialAsset(material, paths, write, &material_bytes).Succeeded()); + const auto material_object = WriteObjectBytes("zematerial", {reinterpret_cast(material_bytes.data()), material_bytes.size()}); + const auto texture_entry = [&](const uuids::uuid& uuid, const CookObjectDescriptor& object) { + return JSON{ + {"kind", "texture"}, + {"uuid", uuids::to_string(uuid)}, + {"format", "image-bmp"}, + {"adapter_revision", 1}, + {"input_key", std::string(64, '1')}, + {"object", ObjectJSON(object)}, + {"dependencies", JSON::array()} + }; + }; + Manifest["assets"] = JSON::array({ + {{"kind", "material"}, {"uuid", uuids::to_string(material_id)}, {"format", "zematerial-v1-json"}, {"adapter_revision", 1}, {"input_key", std::string(64, '1')}, {"object", ObjectJSON(material_object)}, {"dependencies", JSON::array({{{"kind", "texture"}, {"uuid", uuids::to_string(first_texture)}, {"slot", "albedo"}}})}}, + texture_entry(first_texture, first_object), texture_entry(other_texture, other_object) + }); + Manifest["scenes"][0]["roots"] = JSON::array({ + {{"kind", "material"}, {"uuid", uuids::to_string(material_id)}}, + { {"kind", "texture"}, {"uuid", uuids::to_string(first_texture)}}, + { {"kind", "texture"}, {"uuid", uuids::to_string(other_texture)}} + }); + WriteManifest(Manifest); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + Core::Memory::SlabAllocationRoute route = {}; + ASSERT_TRUE(route.Initialize(&Memory.MainArena, ZMega(1))); + const Managers::CookedCpuAssetLimits limits = {.MaxAssets = 3, .MaxObjectBytes = ZKilo(4), .MaxMeshWorkspaceBytes = ZKilo(64)}; + Cache cache = {}; + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 3, 2, 3)); + ClosureCacheDrain drain{cache}; + Cache::Lease valid = {}, invalid = {}, invalid_version = {}; + ASSERT_TRUE(cache.Request(std::move(Package), &valid)); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result valid_result = {}; + ASSERT_TRUE(valid.Poll(&valid_result)); + ASSERT_EQ(valid_result.StateValue, Cache::State::Ready); + ASSERT_NE(valid_result.Assets->FindMaterial(material_key), nullptr); + EXPECT_EQ(valid_result.Assets->FindMaterial(material_key)->Textures[0].UUID, first_texture); + + Manifest["assets"][0]["dependencies"][0]["uuid"] = uuids::to_string(other_texture); + WriteManifest(Manifest); + CookedPackageSnapshot changed = {}; + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), SceneId, Schemas, Limits, Allocator, &Diagnostics, &changed).Succeeded()); + ASSERT_TRUE(cache.Request(std::move(changed), &invalid)); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result invalid_result = {}; + ASSERT_TRUE(invalid.Poll(&invalid_result)); + EXPECT_EQ(invalid_result.StateValue, Cache::State::Failed); + EXPECT_EQ(invalid_result.Error, VFS::VFSError::Corrupted); + Manifest["assets"][0]["dependencies"][0]["uuid"] = uuids::to_string(first_texture); + Manifest["assets"][1]["object"] = ObjectJSON(other_object); + WriteManifest(Manifest); + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), SceneId, Schemas, Limits, Allocator, &Diagnostics, &changed).Succeeded()); + ASSERT_TRUE(cache.Request(std::move(changed), &invalid_version)); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(invalid_version.Poll(&invalid_result)); + EXPECT_EQ(invalid_result.StateValue, Cache::State::Failed); + EXPECT_EQ(invalid_result.Error, VFS::VFSError::Corrupted); + ASSERT_TRUE(valid.Poll(&valid_result)); + EXPECT_EQ(valid_result.Assets->FindMaterial(material_key)->Textures[0].UUID, first_texture); + valid.Reset(); + invalid.Reset(); + invalid_version.Reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.Shutdown()); + } + + TEST_F(CookedPackageSnapshotTest, VersionPoolInitializationReleasesPartialAllocations) + { + using Cache = Managers::CookedCpuClosureCache; + Core::Memory::SlabAllocationRoute base = {}; + ASSERT_TRUE(base.Initialize(&Memory.MainArena, ZMega(1))); + struct RefusingRoute final : Core::Memory::IAllocationRoute + { + Core::Memory::SlabAllocationRoute* Base = nullptr; + uint32_t Calls = 0; + uint32_t RefuseAt = 4; + void* Allocate(size_t bytes) noexcept override + { + return ++Calls == RefuseAt ? nullptr : Base->Allocate(bytes); + } + void* Reallocate(void* pointer, size_t old_size, size_t new_size) noexcept override + { + return Base->Reallocate(pointer, old_size, new_size); + } + void Free(void* pointer) noexcept override + { + Base->Free(pointer); + } + Core::Memory::IWorkingMemoryBudget* WorkingMemoryBudget() noexcept override + { + return Base->WorkingMemoryBudget(); + } + } route; + route.Base = &base; + const Managers::CookedCpuAssetLimits limits = {.MaxAssets = 1, .MaxObjectBytes = 58, .MaxMeshWorkspaceBytes = ZKilo(64)}; + Cache cache = {}; + EXPECT_FALSE(cache.Initialize(route, Schemas, limits, 2, 2, 4)); + EXPECT_EQ(route.Calls, 4u); + EXPECT_EQ(base.AllocatedBytes(), 0u); + route.RefuseAt = UINT32_MAX; + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 2, 2, 4)); + ASSERT_TRUE(cache.Shutdown()); + EXPECT_EQ(base.AllocatedBytes(), 0u); + } + + TEST_F(CookedPackageSnapshotTest, VersionPoolRemovesHashCollisionsWithoutLosingOtherVersions) + { + using Cache = Managers::CookedCpuClosureCache; + const auto texture = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000002"); + const auto other_scene = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000009"); + const CookAssetIdentity identity = {CookAssetKind::Texture, texture}; + const auto uuid_bytes = texture.as_bytes(); + const auto seed = rapidhash_withSeed(uuid_bytes.data(), uuid_bytes.size(), static_cast(CookAssetKind::Texture)); + std::array seen = {-1, -1, -1, -1}; + int first_pixel = -1, second_pixel = -1; + for (int value = 0; value <= 255; ++value) + { + const auto bytes = Pixel(static_cast(value)); + Core::Hash::SHA256 hash = {}; + Digest256 digest = {}; + ASSERT_TRUE(hash.Update({bytes.data(), bytes.size()}).Succeeded()); + ASSERT_TRUE(hash.Finalize(&digest).Succeeded()); + const auto bucket = static_cast(rapidhash_withSeed(digest.Bytes.data(), digest.Bytes.size(), seed) & 3u); + if (seen[bucket] >= 0) + { + first_pixel = seen[bucket]; + second_pixel = value; + break; + } + seen[bucket] = value; + } + ASSERT_GE(first_pixel, 0); + ASSERT_GE(second_pixel, 0); + Core::Memory::SlabAllocationRoute route = {}; + ASSERT_TRUE(route.Initialize(&Memory.MainArena, ZMega(1))); + const Managers::CookedCpuAssetLimits limits = {.MaxAssets = 1, .MaxObjectBytes = 58, .MaxMeshWorkspaceBytes = ZKilo(64)}; + Cache cache = {}; + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 3, 2, 2)); + ClosureCacheDrain drain{cache}; + const auto baseline = route.AllocatedBytes(); + for (bool release_tail : {true, false}) + { + (void) SetTexture(texture, static_cast(first_pixel)); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + Cache::Lease first = {}, second = {}, shared = {}; + ASSERT_TRUE(cache.Request(std::move(Package), &first)); + ASSERT_TRUE(cache.PumpOne()); + (void) SetTexture(texture, static_cast(second_pixel)); + CookedPackageSnapshot next = {}; + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), SceneId, Schemas, Limits, Allocator, &Diagnostics, &next).Succeeded()); + ASSERT_TRUE(cache.Request(std::move(next), &second)); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result first_result = {}, second_result = {}; + ASSERT_TRUE(first.Poll(&first_result)); + ASSERT_TRUE(second.Poll(&second_result)); + ASSERT_EQ(first_result.StateValue, Cache::State::Ready); + ASSERT_EQ(second_result.StateValue, Cache::State::Ready); + const auto* survivor = release_tail ? second_result.Assets->FindTexture(identity) : first_result.Assets->FindTexture(identity); + ASSERT_NE(survivor, nullptr); + if (release_tail) + { + first.Reset(); + } + else + { + second.Reset(); + (void) SetTexture(texture, static_cast(first_pixel)); + } + ASSERT_TRUE(cache.PumpOne()); + if (Manifest["scenes"].size() == 1) + { + auto scene = Manifest["scenes"][0]; + scene["uuid"] = uuids::to_string(other_scene); + scene["source"] = "scenes/other.yaml"; + Manifest["scenes"].push_back(scene); + WriteManifest(Manifest); + } + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), other_scene, Schemas, Limits, Allocator, &Diagnostics, &next).Succeeded()); + ASSERT_TRUE(cache.Request(std::move(next), &shared)); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result shared_result = {}; + ASSERT_TRUE(shared.Poll(&shared_result)); + ASSERT_EQ(shared_result.StateValue, Cache::State::Ready); + EXPECT_EQ(shared_result.Assets->FindTexture(identity), survivor); + first.Reset(); + second.Reset(); + shared.Reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.PumpOne()); + EXPECT_EQ(route.AllocatedBytes(), baseline); + } + ASSERT_TRUE(cache.Shutdown()); + } + + TEST_F(CookedPackageSnapshotTest, VersionPoolIncludesTypedIdentityEvenWhenObjectBytesMatch) + { + using Cache = Managers::CookedCpuClosureCache; + const auto first_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000002"); + const auto second_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000003"); + const auto other_scene = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000009"); + (void) SetTexture(first_id, 10); + auto second_asset = Manifest["assets"][0]; + second_asset["uuid"] = uuids::to_string(second_id); + Manifest["assets"].push_back(second_asset); + auto scene = Manifest["scenes"][0]; + scene["uuid"] = uuids::to_string(other_scene); + scene["source"] = "scenes/other.yaml"; + scene["roots"][0]["uuid"] = uuids::to_string(second_id); + Manifest["scenes"].push_back(scene); + WriteManifest(Manifest); + auto root = Root(); + ASSERT_TRUE(Capture(std::move(root), SceneId).Succeeded()); + CookedPackageSnapshot other = {}; + root = Root(); + ASSERT_TRUE(CookedPackageSnapshot::Capture(std::move(root), other_scene, Schemas, Limits, Allocator, &Diagnostics, &other).Succeeded()); + Core::Memory::SlabAllocationRoute route = {}; + ASSERT_TRUE(route.Initialize(&Memory.MainArena, ZMega(1))); + const Managers::CookedCpuAssetLimits limits = {.MaxAssets = 1, .MaxObjectBytes = 58, .MaxMeshWorkspaceBytes = ZKilo(64)}; + Cache cache = {}; + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 2, 2, 2)); + ClosureCacheDrain drain{cache}; + Cache::Lease first = {}, second = {}; + ASSERT_TRUE(cache.Request(std::move(Package), &first)); + ASSERT_TRUE(cache.Request(std::move(other), &second)); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result first_result = {}, second_result = {}; + ASSERT_TRUE(first.Poll(&first_result)); + ASSERT_TRUE(second.Poll(&second_result)); + ASSERT_EQ(first_result.StateValue, Cache::State::Ready); + ASSERT_EQ(second_result.StateValue, Cache::State::Ready); + const CookAssetIdentity first_key = {CookAssetKind::Texture, first_id}; + const CookAssetIdentity second_key = {CookAssetKind::Texture, second_id}; + ASSERT_NE(first_result.Assets->FindVersion(first_key), nullptr); + ASSERT_NE(second_result.Assets->FindVersion(second_key), nullptr); + EXPECT_EQ(first_result.Assets->FindVersion(first_key)->ObjectDigest, second_result.Assets->FindVersion(second_key)->ObjectDigest); + ASSERT_NE(first_result.Assets->FindTexture(first_key), nullptr); + ASSERT_NE(second_result.Assets->FindTexture(second_key), nullptr); + EXPECT_NE(first_result.Assets->FindTexture(first_key), second_result.Assets->FindTexture(second_key)); + EXPECT_EQ(first_result.Assets->FindTexture(second_key), nullptr); + EXPECT_EQ(second_result.Assets->FindTexture(first_key), nullptr); + first.Reset(); + second.Reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.Shutdown()); + } +} // namespace ZEngine::Cook diff --git a/docs/design/future-plan/scene-cooker-contracts/manifest-v1.golden.json b/ZEngine/tests/Cook/Fixtures/manifest-v1.golden.json similarity index 100% rename from docs/design/future-plan/scene-cooker-contracts/manifest-v1.golden.json rename to ZEngine/tests/Cook/Fixtures/manifest-v1.golden.json diff --git a/docs/design/future-plan/scene-cooker-contracts/manifest-v1.negative-cases.json b/ZEngine/tests/Cook/Fixtures/manifest-v1.negative-cases.json similarity index 100% rename from docs/design/future-plan/scene-cooker-contracts/manifest-v1.negative-cases.json rename to ZEngine/tests/Cook/Fixtures/manifest-v1.negative-cases.json diff --git a/ZEngine/tests/Cook/Process/SceneCookProcess.cmake b/ZEngine/tests/Cook/Process/SceneCookProcess.cmake index 041e331e2..631490d8a 100644 --- a/ZEngine/tests/Cook/Process/SceneCookProcess.cmake +++ b/ZEngine/tests/Cook/Process/SceneCookProcess.cmake @@ -141,6 +141,36 @@ foreach(root_kind IN LISTS root_kinds) check_inputs("${reverse}") check_manifest("${reverse}") + # The standalone visual smoke needs an authored camera in both semantic + # variants. Check the produced source and package on every CI platform; + # rendered parity remains a separate desktop Vulkan exercise. + foreach(variant blend hierarchy) + set(semantic "${run_root}/semantic-${variant}") + file(MAKE_DIRECTORY "${semantic}") + run_child("write-${variant}-camera" "${semantic}") + file(GLOB semantic_scenes "${semantic}/scenes/*.scene") + list(LENGTH semantic_scenes semantic_scene_count) + if(NOT semantic_scene_count EQUAL 1 OR NOT EXISTS "${semantic}/output/desktop-vulkan-le64/manifest.json") + message(FATAL_ERROR "Missing semantic fixture scene or cooked package: ${semantic}") + endif() + list(GET semantic_scenes 0 semantic_scene) + file(READ "${semantic_scene}" semantic_text) + if(NOT semantic_text MATCHES "CameraComponent" OR NOT semantic_text MATCHES "\"is_main\": true") + message(FATAL_ERROR "Semantic fixture lacks its main camera: ${semantic}") + endif() + if(variant STREQUAL "hierarchy") + if(NOT semantic_text MATCHES "\"parent_uuid\": \"80000000-0000-0000-0000-000000000008\"") + message(FATAL_ERROR "Hierarchy fixture lacks its expected parent: ${semantic}") + endif() + else() + file(READ "${semantic}/assets/material.zematerial" semantic_material) + string(JSON semantic_alpha GET "${semantic_material}" alpha_mode) + if(NOT semantic_alpha STREQUAL "BLEND") + message(FATAL_ERROR "Transparent fixture lacks its blend material: ${semantic}") + endif() + endif() + endforeach() + # A real application-defined component persists a typed texture UUID/float, # but not its runtime cache. It is registered before/after built-ins and gets # different real process-local type IDs in independent producers/consumers. @@ -319,6 +349,11 @@ foreach(root_kind IN LISTS root_kinds) run_child(read "${run_root}/old-snapshot") run_child(read "${reverse}/output/desktop-vulkan-le64") run_child(read-changed "${package}") + execute_process(COMMAND "${HELPER}" cross-version "${run_root}/old-snapshot" "${package}" + RESULT_VARIABLE shared_result OUTPUT_VARIABLE shared_output ERROR_VARIABLE shared_errors TIMEOUT 30) + if(NOT shared_result EQUAL 0 OR NOT shared_output MATCHES "ZE_COOK_SHARED_VERSIONS_VERIFIED") + message(FATAL_ERROR "Source-free cross-version sharing failed (${shared_result})\n${shared_output}\n${shared_errors}") + endif() # CMake pipelines execute both children concurrently. The writer waits for # the reader's acquired-handle marker, then publishes and signals completion. diff --git a/ZEngine/tests/Cook/Process/SceneCookProcess.cpp b/ZEngine/tests/Cook/Process/SceneCookProcess.cpp index 59f0e0a11..97ae907b6 100644 --- a/ZEngine/tests/Cook/Process/SceneCookProcess.cpp +++ b/ZEngine/tests/Cook/Process/SceneCookProcess.cpp @@ -11,16 +11,25 @@ #include #include #include +#include #include #include #include #include #include +#include #include +#include #include #include #include #include +#include +#include +#include +#include +#include +#include #include #include #include @@ -58,6 +67,7 @@ namespace ZEngine::Cook::Tests std::filesystem::path Root = {}; bool Custom = false; uint32_t CustomVersion = 1; + std::filesystem::path SecondRoot = {}; }; ProcessOptions Options = {}; @@ -222,6 +232,18 @@ namespace ZEngine::Cook::Tests // Test-only producer/consumer executable, not a shipping cooker CLI or // runtime/Play integration. Consumer mounts ONLY the published package, with // read-only authority, and never constructs an input provider or reads YAML. + struct ClosureCacheDrain + { + Managers::CookedCpuClosureCache& Cache; + ~ClosureCacheDrain() + { + while (Cache.PumpOne()) + { + } + EXPECT_TRUE(Cache.Shutdown()); + } + }; + class SceneCookProcessTest : public testing::Test { protected: @@ -244,7 +266,7 @@ namespace ZEngine::Cook::Tests void SetUp() override { - Reading = Options.Mode == "read" || Options.Mode == "read-changed" || Options.Mode == "reject-missing" || Options.Mode == "reject-corrupt" || Options.Mode == "snapshot-read" || Options.Mode == "reject-schema" || Options.Mode == "reject-version"; + Reading = Options.Mode == "read" || Options.Mode == "read-changed" || Options.Mode == "cross-version" || Options.Mode == "reject-missing" || Options.Mode == "reject-corrupt" || Options.Mode == "snapshot-read" || Options.Mode == "reject-schema" || Options.Mode == "reject-version"; // Allocate different real runtime type IDs in the reader and second // producer, without resetting global IDs or changing schema keys. const bool reverse = Reading || Options.Mode == "write-reverse"; @@ -357,6 +379,30 @@ namespace ZEngine::Cook::Tests }); scene.AddComponent(entity, {MeshUUID}); scene.AddComponent(entity, {MaterialUUID}); + ECS::EntityID camera = {}; + if ((Options.Mode == "write-camera" || Options.Mode == "write-blend-camera" || Options.Mode == "write-hierarchy-camera" || Options.Mode == "write-switch-camera" || Options.Mode == "write-switch-no-camera" || Options.Mode == "write-switch-unique-material")) + { + camera = scene.CreateEntity(); + scene.AddComponent(camera, {Identity("a0000000-0000-0000-0000-00000000000a")}); + scene.AddComponent( + camera, + { + .Position = {1.f, 2.f, 8.f} + }); + scene.AddComponent(camera, {.IsMain = true}); + } + if (Options.Mode == "write-hierarchy" || Options.Mode == "write-hierarchy-camera") + { + const auto parent = scene.CreateEntity(); + scene.AddComponent(parent, {Identity("80000000-0000-0000-0000-000000000008")}); + scene.AddComponent( + parent, + { + .Position = {1, 0, 0} + }); + scene.GetComponent(entity)->Position.x = 0; + ASSERT_TRUE(scene.SetParent(entity, parent)); + } if (Options.Custom) { scene.AddComponent(entity, {.Texture = TextureUUID, .Weight = .375f, .RuntimeCache = 12345}); @@ -368,6 +414,19 @@ namespace ZEngine::Cook::Tests ECS::SceneDocument document = {}; ASSERT_TRUE(ECS::SceneDocument::Capture(scene, settings, Schemas, &document, &SceneDiagnostics, Limits.SceneLimits)); ASSERT_TRUE(document.Save(Files, Path(SceneFile), &SceneDiagnostics).Succeeded()); + if ((Options.Mode == "write-switch" || Options.Mode == "write-switch-camera" || Options.Mode == "write-switch-no-camera" || Options.Mode == "write-switch-unique-material")) + { + if (Options.Mode == "write-switch-no-camera") + scene.DestroyEntity(camera); + if (Options.Mode == "write-switch-unique-material") + scene.GetComponent(entity)->MaterialUUID = Identity("b0000000-0000-0000-0000-00000000000b"); + scene.GetComponent(entity)->Position.x = 2; + settings.UUID = Identity("90000000-0000-0000-0000-000000000009"); + settings.Name = "Alternate cooked scene"; + ECS::SceneDocument alternate = {}; + ASSERT_TRUE(ECS::SceneDocument::Capture(scene, settings, Schemas, &alternate, &SceneDiagnostics, Limits.SceneLimits)); + ASSERT_TRUE(alternate.Save(Files, Path("/scenes/alternate.scene"), &SceneDiagnostics).Succeeded()); + } scene.Shutdown(); Importers::AssetMesh mesh = {}; @@ -375,6 +434,10 @@ namespace ZEngine::Cook::Tests mesh.Vertices.init(&Memory.MainArena, 24, 24); mesh.Vertices[8] = 1; mesh.Vertices[17] = 1; + if (Options.Mode == "write-blend" || Options.Mode == "write-blend-camera") + { + mesh.Vertices[5] = mesh.Vertices[13] = mesh.Vertices[21] = 1; + } mesh.Indices.init(&Memory.MainArena, 3); mesh.Indices.push(0); mesh.Indices.push(1); @@ -404,13 +467,26 @@ namespace ZEngine::Cook::Tests ASSERT_EQ(Helpers::secure_strcpy(material.Name, sizeof(material.Name), "Fixture material"), Helpers::MEMORY_OP_SUCCESS); material.Textures[0].UUID = TextureUUID; ASSERT_EQ(Helpers::secure_strcpy(material.Textures[0].Path, sizeof(material.Textures[0].Path), "assets/pixel.bmp"), Helpers::MEMORY_OP_SUCCESS); - material.AlbedoColor = {1, .5f, .25f, 1}; - material.Factors[2] = .5f; + material.AlbedoColor = {1, .5f, .25f, 1}; + material.Factors[2] = .5f; + if (Options.Mode == "write-blend" || Options.Mode == "write-blend-camera") + { + material.AlphaMode = Rendering::Materials::AlphaMode::Blend; + material.Factors[0] = .5f; + } Core::Containers::FixedArray paths = {Relative("assets/pixel.bmp")}; MaterialSink sink = {}; ASSERT_TRUE(Codec::WriteCanonicalMaterialAsset(material, paths, MaterialSink::Write, &sink).Succeeded()); Write(Path("/assets/material.zematerial"), {sink.Bytes.data(), sink.Size}); WriteText("/assets/material.zematerial.meta", R"({"uuid":"40000000-0000-0000-0000-000000000004","importer":"MaterialImporter","settings":[]})"); + if (Options.Mode == "write-switch-unique-material") + { + material.UUID = Identity("b0000000-0000-0000-0000-00000000000b"); + MaterialSink alternate_sink = {}; + ASSERT_TRUE(Codec::WriteCanonicalMaterialAsset(material, paths, MaterialSink::Write, &alternate_sink).Succeeded()); + Write(Path("/assets/alternate.zematerial"), {alternate_sink.Bytes.data(), alternate_sink.Size}); + WriteText("/assets/alternate.zematerial.meta", R"({"uuid":"b0000000-0000-0000-0000-00000000000b","importer":"MaterialImporter","settings":[]})"); + } SavePixel(); WriteText("/assets/pixel.bmp.meta", R"({"uuid":"50000000-0000-0000-0000-000000000005","importer":"TextureImporter","settings":[]})"); @@ -578,8 +654,11 @@ namespace ZEngine::Cook::Tests CookAdapterRegistry adapters = {}; const CookAdapterFns functions[] = {CookEnvironmentAdapter(), CookMaterialAdapter(), CookMeshAdapter(), CookTextureAdapter()}; ASSERT_TRUE(adapters.Initialize(functions).Succeeded()); - const VFS::VFSPath scenes[] = {Path(SceneFile)}; - const SceneCookRequest request = {scenes, Path("/output"), CookTarget::DesktopVulkanLE64, force}; + const VFS::VFSPath scenes[] = {Path(SceneFile), Path("/scenes/alternate.scene")}; + const SceneCookRequest request = { + {scenes, (Options.Mode == "write-switch" || Options.Mode == "write-switch-camera" || Options.Mode == "write-switch-no-camera" || Options.Mode == "write-switch-unique-material") ? 2u : 1u}, + Path("/output"), CookTarget::DesktopVulkanLE64, force + }; const SceneCookContext context = {&source, &Files, &Files, Path("/scratch"), &Schemas, Migrations.IsInitialized() ? &Migrations : nullptr, &adapters, &Allocator, Limits}; const Digest256 previous_digest = {.Bytes = {{0xa5, 0x5a}}}; SceneCookResult result = {}; @@ -637,9 +716,9 @@ namespace ZEngine::Cook::Tests return; } ASSERT_TRUE(cooked.Succeeded()) << "Cook failed (VFS " << static_cast(cooked.Error()) << "; writes " << Faults.Writes << ", reads " << Faults.Reads << ", object renames " << Faults.ObjectRenames << ", manifest renames " << Faults.ManifestRenames << ")."; - EXPECT_EQ(result.SceneCount, 1u); - EXPECT_EQ(result.AssetCount, 4u); - EXPECT_EQ(result.ObjectCount, 5u); + EXPECT_EQ(result.SceneCount, (Options.Mode == "write-switch" || Options.Mode == "write-switch-camera" || Options.Mode == "write-switch-no-camera" || Options.Mode == "write-switch-unique-material") ? 2u : 1u); + EXPECT_EQ(result.AssetCount, Options.Mode == "write-switch-unique-material" ? 5u : 4u); + EXPECT_EQ(result.ObjectCount, Options.Mode == "write-switch-unique-material" ? 7u : (Options.Mode == "write-switch" || Options.Mode == "write-switch-camera" || Options.Mode == "write-switch-no-camera" || Options.Mode == "write-switch-unique-material") ? 6u : 5u); EXPECT_EQ(result.Published, published); EXPECT_LE(result.PeakWorkingBytes, Limits.MaxWorkingBytes); EXPECT_FALSE(Diagnostics.HasErrors()); @@ -957,6 +1036,447 @@ namespace ZEngine::Cook::Tests } return VFS::VFSResult::Ok(); } + + // Exercise the runtime readers in a fresh process that mounts only the + // cook output. The source trees have already been removed by the driver. + void CheckRuntimeReaders() + { + ASSERT_TRUE(Schemas.FreezeForWorker(&SceneDiagnostics)); + VFS::VFSReadRootLease root = {}; + ASSERT_TRUE(Files.AcquireReadRoot(VFS::VFSPath::Root(), &root).Succeeded()); + Managers::CookedPackageSnapshot package = {}; + ASSERT_TRUE(Managers::CookedPackageSnapshot::Capture(std::move(root), SceneUUID, Schemas, Limits, Allocator, &Diagnostics, &package).Succeeded()); + CookCodecAllocationRoute route(Allocator); + Managers::CookedObjectSnapshot object = {}; + + ASSERT_TRUE(package.CaptureScene(route, ZKilo(64), &object).Succeeded()); + { + Core::Memory::ArenaAllocator scene_arena = {}; + Memory.MainArena.CreateSubArena(ZMega(16), &scene_arena, "CookRuntimeProcess/Scene"); + PaddedAtomic occupied = {}; + const ECS::SceneStorageSlot slot = {&scene_arena, &occupied}; + ECS::SceneStorageLease storage = {}; + ASSERT_TRUE(ECS::SceneStorageLease::Acquire({&slot, 1}, &storage)); + VFS::VFSReadRootLease duplicate_root = {}; + ASSERT_TRUE(Files.AcquireReadRoot(VFS::VFSPath::Root(), &duplicate_root).Succeeded()); + Managers::CookedPackageSnapshot duplicate = {}; + ASSERT_TRUE(Managers::CookedPackageSnapshot::Capture(std::move(duplicate_root), SceneUUID, Schemas, Limits, Allocator, &Diagnostics, &duplicate).Succeeded()); + const auto digest = duplicate.ManifestDigest(); + Managers::CookedCpuClosureCache cache = {}; + const Managers::CookedCpuAssetLimits asset_limits = {.MaxAssets = 4, .MaxObjectBytes = ZKilo(64), .MaxMeshWorkspaceBytes = ZKilo(64)}; + ASSERT_TRUE(cache.Initialize(route, Schemas, asset_limits, 1, 2, 8)); + ClosureCacheDrain drain{cache}; + ECS::CookedSceneReconstruction reconstruction = {}; + std::atomic stop = true; + EXPECT_EQ(ECS::ReconstructCookedScene(duplicate, std::move(storage), Schemas, route, ZKilo(64), ZMega(4), cache, &reconstruction, &SceneDiagnostics, Limits.SceneLimits, &stop), ECS::CookedSceneReconstructionError::Cancelled); + EXPECT_NE(storage.Arena(), nullptr); + EXPECT_TRUE(duplicate.IsValid()); + stop = false; + EXPECT_EQ(ECS::ReconstructCookedScene(duplicate, std::move(storage), Schemas, route, 1, ZMega(4), cache, &reconstruction, &SceneDiagnostics, Limits.SceneLimits, &stop), ECS::CookedSceneReconstructionError::SceneObject); + EXPECT_NE(storage.Arena(), nullptr); + EXPECT_TRUE(duplicate.IsValid()); + EXPECT_EQ(reconstruction.Instance, nullptr); + EXPECT_FALSE(reconstruction.Assets); + ASSERT_EQ(ECS::ReconstructCookedScene(duplicate, std::move(storage), Schemas, route, ZKilo(64), ZMega(4), cache, &reconstruction, &SceneDiagnostics, Limits.SceneLimits, &stop), ECS::CookedSceneReconstructionError::None); + ASSERT_NE(reconstruction.Instance, nullptr); + EXPECT_FALSE(duplicate.IsValid()); + EXPECT_TRUE(reconstruction.Instance->MatchesCookedClosure(digest, SceneUUID)); + EXPECT_FALSE(reconstruction.Instance->MatchesCookedClosure(digest, MeshUUID)); + EXPECT_FALSE(reconstruction.Instance->MatchesCookedClosure({}, SceneUUID)); + EXPECT_EQ(reconstruction.Instance->World().AliveCount(), 1u); + Managers::CookedCpuClosureCache::Result result = {}; + ASSERT_TRUE(reconstruction.Assets.Poll(&result)); + EXPECT_EQ(result.StateValue, Managers::CookedCpuClosureCache::State::Pending); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(reconstruction.Assets.Poll(&result)); + ASSERT_EQ(result.StateValue, Managers::CookedCpuClosureCache::State::Ready); + EXPECT_EQ(result.Assets->ManifestDigest(), digest); + EXPECT_EQ(result.Assets->SceneUUID(), SceneUUID); + reconstruction.Assets.Reset(); + reconstruction.Instance.reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.Shutdown()); + EXPECT_FALSE(occupied.value.load()); + } + ECS::SceneDocument document = {}; + ASSERT_TRUE(ECS::BinarySceneCodec::Decode(object.Bytes(), Schemas, &document, &SceneDiagnostics, Limits.SceneLimits, &route)); + ASSERT_TRUE(package.ValidateScene(document, Schemas, &SceneDiagnostics).Succeeded()); + std::atomic cancelled = true; + EXPECT_EQ(package.ValidateScene(document, Schemas, &SceneDiagnostics, &cancelled).Error(), VFS::VFSError::Cancelled); + ECS::Scene empty = {}; + empty.Initialize(&Memory.MainArena); + ECS::SceneDocument missing_roots = {}; + const ECS::SceneDocumentSettingsView empty_settings = {.UUID = SceneUUID, .Name = SceneName}; + ASSERT_TRUE(ECS::SceneDocument::Capture(empty, empty_settings, Schemas, &missing_roots, &SceneDiagnostics)); + EXPECT_EQ(package.ValidateScene(missing_roots, Schemas, &SceneDiagnostics).Error(), VFS::VFSError::Corrupted); + empty.Shutdown(); + object.Reset(); + + const CookAssetIdentity mesh_id = {CookAssetKind::Mesh, MeshUUID}; + ASSERT_TRUE(package.CaptureAsset(mesh_id, route, ZKilo(64), &object).Succeeded()); + size_t workspace_bytes = 0; + ASSERT_TRUE(Codec::MeasureMeshAssetReadWorkspace(object.Source(), &workspace_bytes, MeshUUID).Succeeded()); + CookWorkspace workspace = {}; + ASSERT_TRUE(workspace.Initialize(Allocator, workspace_bytes, CookMemoryCategory::Codecs).Succeeded()); + Importers::AssetMesh mesh = {}; + Importers::AssetNodeHierarchy hierarchy = {}; + ASSERT_TRUE(Managers::CookedAssetReader::ReadMesh(package, mesh_id, object, workspace.Arena(), *route.WorkingMemoryBudget(), &mesh, &hierarchy).Succeeded()); + ASSERT_EQ(mesh.SubMeshes.size(), 1u); + EXPECT_EQ(mesh.SubMeshes[0].MaterialUUID, MaterialUUID); + EXPECT_EQ(hierarchy.MeshUUID, MeshUUID); + EXPECT_EQ(Managers::CookedAssetReader::ReadMesh(package, {CookAssetKind::Mesh, MaterialUUID}, object, workspace.Arena(), *route.WorkingMemoryBudget(), &mesh, &hierarchy).Error(), VFS::VFSError::Corrupted); + object.Reset(); + + const CookAssetIdentity material_id = {CookAssetKind::Material, MaterialUUID}; + ASSERT_TRUE(package.CaptureAsset(material_id, route, ZKilo(64), &object).Succeeded()); + Codec::MaterialAssetReader reader = {}; + const Codec::MaterialAssetDocument* material = nullptr; + ASSERT_TRUE(Managers::CookedAssetReader::ReadMaterial(package, material_id, object, reader, &material).Succeeded()); + ASSERT_NE(material, nullptr); + EXPECT_EQ(material->Textures[0].UUID, TextureUUID); + object.Reset(); + + const CookAssetIdentity texture_id = {CookAssetKind::Texture, TextureUUID}; + ASSERT_TRUE(package.CaptureAsset(texture_id, route, ZKilo(64), &object).Succeeded()); + Importers::ImageCodec::EncodedImageDescription image = {}; + ASSERT_TRUE(Managers::CookedAssetReader::ValidateTexture(package, texture_id, object, route, {}, &image).Succeeded()); + EXPECT_EQ(image.Kind, Importers::ImageCodec::EncodedImageKind::BMP); + EXPECT_EQ(image.Width, 1u); + { + Importers::ImageCodec::DecodedImage pixels = {}; + ASSERT_TRUE(Managers::CookedAssetReader::ReadTexture(package, texture_id, object, route, {}, &pixels).Succeeded()); + ASSERT_EQ(pixels.Pixels().size(), 4u); + EXPECT_EQ(pixels.Pixels()[0], 30u); + EXPECT_EQ(pixels.Pixels()[1], 20u); + EXPECT_EQ(pixels.Pixels()[2], Options.Mode == "read-changed" ? 90u : 10u); + EXPECT_EQ(pixels.Pixels()[3], 255u); + } + EXPECT_EQ(Managers::CookedAssetReader::ReadMaterial(package, material_id, object, reader, &material).Error(), VFS::VFSError::Corrupted); + object.Reset(); + + const CookAssetIdentity environment_id = {CookAssetKind::EnvironmentMap, SkyUUID}; + ASSERT_TRUE(package.CaptureAsset(environment_id, route, ZKilo(64), &object).Succeeded()); + Codec::EnvironmentMapFileHeader environment = {}; + ASSERT_TRUE(Managers::CookedAssetReader::ReadEnvironment(package, environment_id, object, Stream, &environment).Succeeded()); + EXPECT_EQ(environment.LayerCount, 6u); + EXPECT_EQ(environment.BufferByteSize, 96u); + EXPECT_EQ(route.Error(), VFS::VFSError::OK); + object.Reset(); + + const auto baseline = Allocator.OutstandingAllocations(); + Managers::CookedCpuAssetLimits limits = {.MaxAssets = 4, .MaxObjectBytes = ZKilo(64), .MaxMeshWorkspaceBytes = ZKilo(64)}; + Managers::CookedCpuAssetSet assets = {}; + auto refused = limits; + refused.MaxAssets = 3; + EXPECT_EQ(Managers::CookedCpuAssetSet::Capture(package, route, refused, &assets).Error(), VFS::VFSError::SizeLimitExceeded); + EXPECT_FALSE(assets.IsValid()); + EXPECT_EQ(Allocator.OutstandingAllocations(), baseline); + refused = limits; + refused.MaxMeshWorkspaceBytes = 1; + EXPECT_EQ(Managers::CookedCpuAssetSet::Capture(package, route, refused, &assets).Error(), VFS::VFSError::SizeLimitExceeded); + EXPECT_FALSE(assets.IsValid()); + EXPECT_EQ(Allocator.OutstandingAllocations(), baseline); + EXPECT_EQ(Managers::CookedCpuAssetSet::Capture(package, route, limits, &assets, &cancelled).Error(), VFS::VFSError::Cancelled); + EXPECT_EQ(Allocator.OutstandingAllocations(), baseline); + struct CancellingRoute final : Core::Memory::IAllocationRoute + { + CookCodecAllocationRoute* Inner = nullptr; + std::atomic* Cancelled = nullptr; + uint32_t Calls = 0; + void* Allocate(size_t size) noexcept override + { + void* memory = Inner->Allocate(size); + if (memory && ++Calls == 2) + { + Cancelled->store(true, std::memory_order_release); + } + return memory; + } + void* Reallocate(void* pointer, size_t old_size, size_t new_size) noexcept override + { + return Inner->Reallocate(pointer, old_size, new_size); + } + void Free(void* pointer) noexcept override + { + Inner->Free(pointer); + } + Core::Memory::IWorkingMemoryBudget* WorkingMemoryBudget() noexcept override + { + return Inner->WorkingMemoryBudget(); + } + } cancelling; + cancelled.store(false, std::memory_order_release); + cancelling.Inner = &route; + cancelling.Cancelled = &cancelled; + EXPECT_EQ(Managers::CookedCpuAssetSet::Capture(package, cancelling, limits, &assets, &cancelled).Error(), VFS::VFSError::Cancelled); + EXPECT_EQ(Allocator.OutstandingAllocations(), baseline); + struct RefusingRoute final : Core::Memory::IAllocationRoute + { + CookCodecAllocationRoute* Inner = nullptr; + uint32_t Calls = 0; + void* Allocate(size_t size) noexcept override + { + return ++Calls == 3 ? nullptr : Inner->Allocate(size); + } + void* Reallocate(void* pointer, size_t old_size, size_t new_size) noexcept override + { + return Inner->Reallocate(pointer, old_size, new_size); + } + void Free(void* pointer) noexcept override + { + Inner->Free(pointer); + } + Core::Memory::IWorkingMemoryBudget* WorkingMemoryBudget() noexcept override + { + return Inner->WorkingMemoryBudget(); + } + } refusing; + refusing.Inner = &route; + EXPECT_EQ(Managers::CookedCpuAssetSet::Capture(package, refusing, limits, &assets).Error(), VFS::VFSError::OutOfMemory); + EXPECT_FALSE(assets.IsValid()); + EXPECT_EQ(Allocator.OutstandingAllocations(), baseline); + ASSERT_TRUE(Managers::CookedCpuAssetSet::Capture(package, route, limits, &assets).Succeeded()); + const auto with_assets = Allocator.OutstandingAllocations(); + ASSERT_EQ(assets.Size(), 4u); + EXPECT_EQ(assets.ManifestDigest(), package.ManifestDigest()); + ASSERT_NE(assets.FindVersion(mesh_id), nullptr); + EXPECT_EQ(assets.FindVersion(mesh_id)->ObjectDigest, package.FindSelectedAsset(mesh_id)->Result.Object.Digest); + ASSERT_NE(assets.FindMesh(mesh_id), nullptr); + EXPECT_EQ(assets.FindMesh(mesh_id)->Vertices.size(), 24u); + ASSERT_NE(assets.FindHierarchy(mesh_id), nullptr); + EXPECT_EQ(assets.FindHierarchy(mesh_id)->MeshUUID, MeshUUID); + ASSERT_NE(assets.FindMaterial(material_id), nullptr); + EXPECT_EQ(assets.FindMaterial(material_id)->Textures[0].UUID, TextureUUID); + ASSERT_NE(assets.FindTexture(texture_id), nullptr); + EXPECT_EQ(assets.FindTexture(texture_id)->Pixels()[2], Options.Mode == "read-changed" ? 90u : 10u); + Rendering::Materials::CookedTextureBinding binding = { + texture_id, assets.FindVersion(texture_id)->ObjectDigest, {7, 1} + }; + Rendering::Meshes::MeshMaterial gpu_material = {}; + ASSERT_TRUE(Rendering::Materials::BuildCookedMaterial(assets, material_id, {&binding, 1}, &gpu_material)); + EXPECT_EQ(gpu_material.AlbedoMap, 7u); + EXPECT_EQ(gpu_material.NormalMap, INVALID_MAP_HANDLE); + EXPECT_FLOAT_EQ(gpu_material.AlbedoColor.y, .5f); + EXPECT_FLOAT_EQ(gpu_material.Factors.z, .5f); + auto invalid_binding = binding; + invalid_binding.ObjectDigest.Bytes[0] ^= 1; + gpu_material.AlbedoMap = 123; + EXPECT_FALSE(Rendering::Materials::BuildCookedMaterial(assets, material_id, {&invalid_binding, 1}, &gpu_material)); + EXPECT_EQ(gpu_material.AlbedoMap, 123u); + EXPECT_FALSE(Rendering::Materials::BuildCookedMaterial(assets, material_id, {}, &gpu_material)); + invalid_binding = binding; + invalid_binding.Handle.Index = INVALID_MAP_HANDLE; + EXPECT_FALSE(Rendering::Materials::BuildCookedMaterial(assets, material_id, {&invalid_binding, 1}, &gpu_material)); + ASSERT_NE(assets.FindEnvironment(environment_id), nullptr); + EXPECT_EQ(assets.FindEnvironment(environment_id)->LayerCount, 6u); + EXPECT_EQ(assets.EnvironmentObject(environment_id).size(), 168u); + EXPECT_EQ(assets.FindMesh(texture_id), nullptr); + // Cancel from inside the actual cache build allocation path. A + // shared build must continue; an exclusive build may stop, but its + // lease remains valid until the owner releases it. + for (bool shared : {true, false}) + { + struct CacheCancellationRoute final : Core::Memory::IAllocationRoute + { + CookCodecAllocationRoute* Inner = nullptr; + Managers::CookedCpuClosureCache* Cache = nullptr; + Managers::CookedCpuClosureCache::CancellationToken Token = {}; + Managers::CookedCpuClosureCache::CancellationToken ForeignToken = {}; + bool Armed = false; + bool Attempted = false; + bool Cancelled = false; + bool ForeignCancelled = false; + void* Allocate(size_t size) noexcept override + { + if (Armed && !Attempted) + { + Attempted = true; + ForeignCancelled = Cache->CancelUnsharedBuild(ForeignToken); + Cancelled = Cache->CancelUnsharedBuild(Token); + } + return Inner->Allocate(size); + } + void* Reallocate(void* pointer, size_t old_size, size_t new_size) noexcept override + { + return Inner->Reallocate(pointer, old_size, new_size); + } + void Free(void* pointer) noexcept override + { + Inner->Free(pointer); + } + Core::Memory::IWorkingMemoryBudget* WorkingMemoryBudget() noexcept override + { + return Inner->WorkingMemoryBudget(); + } + } cancellation_route; + cancellation_route.Inner = &route; + Managers::CookedCpuClosureCache foreign = {}; + ASSERT_TRUE(foreign.Initialize(route, Schemas, limits, 2, 2, 8)); + ClosureCacheDrain foreign_drain{foreign}; + VFS::VFSReadRootLease foreign_root = {}; + ASSERT_TRUE(Files.AcquireReadRoot(VFS::VFSPath::Root(), &foreign_root).Succeeded()); + Managers::CookedPackageSnapshot foreign_input = {}; + ASSERT_TRUE(Managers::CookedPackageSnapshot::Capture(std::move(foreign_root), SceneUUID, Schemas, Limits, Allocator, &Diagnostics, &foreign_input).Succeeded()); + Managers::CookedCpuClosureCache::Lease foreign_lease = {}; + ASSERT_TRUE(foreign.Request(std::move(foreign_input), &foreign_lease)); + Managers::CookedCpuClosureCache cache = {}; + cancellation_route.Cache = &cache; + ASSERT_TRUE(cache.Initialize(cancellation_route, Schemas, limits, 2, 2, 8)); + ClosureCacheDrain drain{cache}; + VFS::VFSReadRootLease root = {}; + ASSERT_TRUE(Files.AcquireReadRoot(VFS::VFSPath::Root(), &root).Succeeded()); + Managers::CookedPackageSnapshot input = {}; + ASSERT_TRUE(Managers::CookedPackageSnapshot::Capture(std::move(root), SceneUUID, Schemas, Limits, Allocator, &Diagnostics, &input).Succeeded()); + Managers::CookedCpuClosureCache::Lease first = {}, second = {}; + ASSERT_TRUE(cache.Request(std::move(input), &first)); + if (shared) + ASSERT_TRUE(first.Retain(&second)); + cancellation_route.Token = first.CancellationIdentity(); + cancellation_route.ForeignToken = foreign_lease.CancellationIdentity(); + ASSERT_EQ(cancellation_route.Token.Slot, cancellation_route.ForeignToken.Slot); + ASSERT_EQ(cancellation_route.Token.Serial, cancellation_route.ForeignToken.Serial); + cancellation_route.Armed = true; + ASSERT_TRUE(cache.PumpOne()); + EXPECT_TRUE(cancellation_route.Attempted); + EXPECT_FALSE(cancellation_route.ForeignCancelled); + EXPECT_EQ(cancellation_route.Cancelled, !shared); + Managers::CookedCpuClosureCache::Result result = {}; + ASSERT_TRUE(first.Poll(&result)); + EXPECT_EQ(result.StateValue, shared ? Managers::CookedCpuClosureCache::State::Ready : Managers::CookedCpuClosureCache::State::Failed); + if (!shared) + EXPECT_EQ(result.Error, VFS::VFSError::Cancelled); + second.Reset(); + first.Reset(); + ASSERT_TRUE(cache.PumpOne()); + EXPECT_TRUE(cache.Shutdown()); + foreign_lease.Reset(); + ASSERT_TRUE(foreign.PumpOne()); + EXPECT_TRUE(foreign.Shutdown()); + } + { + VFS::VFSReadRootLease duplicate_root = {}; + ASSERT_TRUE(Files.AcquireReadRoot(VFS::VFSPath::Root(), &duplicate_root).Succeeded()); + Managers::CookedPackageSnapshot duplicate = {}; + ASSERT_TRUE(Managers::CookedPackageSnapshot::Capture(std::move(duplicate_root), SceneUUID, Schemas, Limits, Allocator, &Diagnostics, &duplicate).Succeeded()); + Managers::CookedCpuClosureCache cache = {}; + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 2, 2, 8)); + ClosureCacheDrain drain{cache}; + Managers::CookedCpuClosureCache::Lease first = {}, second = {}; + ASSERT_TRUE(cache.Request(std::move(duplicate), &first)); + ASSERT_TRUE(cache.Request(std::move(package), &second)); + EXPECT_TRUE(package.IsValid()); + ASSERT_TRUE(cache.PumpOne()); + Managers::CookedCpuClosureCache::Result first_result = {}, second_result = {}; + ASSERT_TRUE(first.Poll(&first_result)); + ASSERT_TRUE(second.Poll(&second_result)); + ASSERT_EQ(first_result.StateValue, Managers::CookedCpuClosureCache::State::Ready); + EXPECT_EQ(first_result.Assets, second_result.Assets); + EXPECT_EQ(first_result.Assets->SceneUUID(), SceneUUID); + EXPECT_EQ(first_result.Assets->ManifestDigest(), package.ManifestDigest()); + ASSERT_NE(first_result.Assets->FindTexture(texture_id), nullptr); + EXPECT_EQ(first_result.Assets->FindTexture(texture_id)->Pixels()[2], Options.Mode == "read-changed" ? 90u : 10u); + Rendering::Meshes::CookedMeshStaging mesh_staging = {}; + EXPECT_FALSE(mesh_staging.Capture(std::move(second), texture_id)); + EXPECT_TRUE(second); + ASSERT_TRUE(mesh_staging.Capture(std::move(first), mesh_id)); + EXPECT_FALSE(first); + EXPECT_EQ(mesh_staging.Identity(), mesh_id); + EXPECT_EQ(mesh_staging.Digest(), first_result.Assets->FindVersion(mesh_id)->ObjectDigest); + EXPECT_EQ(mesh_staging.VertexBytes(), 24u * sizeof(float)); + EXPECT_EQ(mesh_staging.IndexBytes(), 3u * sizeof(uint32_t)); + ASSERT_NE(mesh_staging.Mesh(), nullptr); + EXPECT_EQ(mesh_staging.Mesh()->SubMeshes[0].MaterialUUID, MaterialUUID); + EXPECT_FALSE(mesh_staging.StageOne()); + EXPECT_FALSE(cache.Shutdown()); + mesh_staging.Reset(); + ASSERT_TRUE(second.Retain(&first)); + Rendering::Textures::CookedEnvironmentUpload environment = {}; + EXPECT_FALSE(environment.Capture(std::move(first), texture_id)); + EXPECT_TRUE(first); + ASSERT_TRUE(environment.Capture(std::move(first), environment_id)); + EXPECT_FALSE(first); + EXPECT_EQ(environment.Identity(), environment_id); + EXPECT_EQ(environment.Digest(), second_result.Assets->FindVersion(environment_id)->ObjectDigest); + ASSERT_NE(environment.Header(), nullptr); + EXPECT_EQ(environment.Header()->FaceWidth, 1u); + EXPECT_EQ(environment.Header()->BufferByteSize, 96u); + EXPECT_FALSE(environment.StageOne()); + environment.Reset(); + first.Reset(); + second.Reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.Shutdown()); + } + EXPECT_EQ(Allocator.OutstandingAllocations(), with_assets); + Managers::CookedCpuAssetSet moved = std::move(assets); + EXPECT_FALSE(assets.IsValid()); + package.Reset(); + const auto after_package_reset = Allocator.OutstandingAllocations(); + EXPECT_EQ(moved.FindMesh(mesh_id)->SubMeshes[0].MaterialUUID, MaterialUUID); + EXPECT_EQ(moved.FindTexture(texture_id)->Pixels()[0], 30u); + moved.Reset(); + EXPECT_EQ(Allocator.OutstandingAllocations(), after_package_reset - (with_assets - baseline)); + } + + void CheckSharedVersionsAcrossPackages() + { + ASSERT_TRUE(Schemas.FreezeForWorker(&SceneDiagnostics)); + VFS::VFSDiskBackend next_disk = {}; + const auto next_path = Options.SecondRoot.generic_u8string(); + next_disk.Initialize(reinterpret_cast(next_path.c_str()), VFS::VFSBackendCaps::Read | VFS::VFSBackendCaps::List, &Memory.MainArena); + VFS::VFSContext next_files = {}; + next_files.Initialize(&Memory.MainArena); + ASSERT_TRUE(next_files.Mount(&next_disk, VFS::VFSPath::Root(), 0).Succeeded()); + { + VFS::VFSReadRootLease old_root = {}, next_root = {}; + ASSERT_TRUE(Files.AcquireReadRoot(VFS::VFSPath::Root(), &old_root).Succeeded()); + ASSERT_TRUE(next_files.AcquireReadRoot(VFS::VFSPath::Root(), &next_root).Succeeded()); + Managers::CookedPackageSnapshot old_package = {}, next_package = {}; + ASSERT_TRUE(Managers::CookedPackageSnapshot::Capture(std::move(old_root), SceneUUID, Schemas, Limits, Allocator, &Diagnostics, &old_package).Succeeded()); + ASSERT_TRUE(Managers::CookedPackageSnapshot::Capture(std::move(next_root), SceneUUID, Schemas, Limits, Allocator, &Diagnostics, &next_package).Succeeded()); + EXPECT_NE(old_package.ManifestDigest(), next_package.ManifestDigest()); + CookCodecAllocationRoute route(Allocator); + const Managers::CookedCpuAssetLimits limits = {.MaxAssets = 4, .MaxObjectBytes = ZKilo(64), .MaxMeshWorkspaceBytes = ZKilo(64)}; + Managers::CookedCpuClosureCache cache = {}; + ASSERT_TRUE(cache.Initialize(route, Schemas, limits, 2, 2, 8)); + ClosureCacheDrain drain{cache}; + Managers::CookedCpuClosureCache::Lease old_lease = {}, next_lease = {}; + ASSERT_TRUE(cache.Request(std::move(old_package), &old_lease)); + ASSERT_TRUE(cache.Request(std::move(next_package), &next_lease)); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.PumpOne()); + Managers::CookedCpuClosureCache::Result old_result = {}, next_result = {}; + ASSERT_TRUE(old_lease.Poll(&old_result)); + ASSERT_TRUE(next_lease.Poll(&next_result)); + ASSERT_EQ(old_result.StateValue, Managers::CookedCpuClosureCache::State::Ready); + ASSERT_EQ(next_result.StateValue, Managers::CookedCpuClosureCache::State::Ready); + const CookAssetIdentity mesh = {CookAssetKind::Mesh, MeshUUID}; + const CookAssetIdentity material = {CookAssetKind::Material, MaterialUUID}; + const CookAssetIdentity texture = {CookAssetKind::Texture, TextureUUID}; + const CookAssetIdentity environment = {CookAssetKind::EnvironmentMap, SkyUUID}; + ASSERT_NE(old_result.Assets->FindMesh(mesh), nullptr); + ASSERT_NE(old_result.Assets->FindEnvironment(environment), nullptr); + ASSERT_NE(old_result.Assets->FindMaterial(material), nullptr); + ASSERT_NE(next_result.Assets->FindMaterial(material), nullptr); + ASSERT_NE(old_result.Assets->FindTexture(texture), nullptr); + ASSERT_NE(next_result.Assets->FindTexture(texture), nullptr); + EXPECT_EQ(old_result.Assets->FindMesh(mesh), next_result.Assets->FindMesh(mesh)); + EXPECT_EQ(old_result.Assets->FindEnvironment(environment), next_result.Assets->FindEnvironment(environment)); + EXPECT_NE(old_result.Assets->FindMaterial(material), next_result.Assets->FindMaterial(material)); + EXPECT_NE(old_result.Assets->FindTexture(texture), next_result.Assets->FindTexture(texture)); + EXPECT_EQ(old_result.Assets->FindTexture(texture)->Pixels()[2], 10u); + EXPECT_EQ(next_result.Assets->FindTexture(texture)->Pixels()[2], 90u); + old_lease.Reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(next_lease.Poll(&next_result)); + EXPECT_EQ(next_result.Assets->FindMesh(mesh)->SubMeshes[0].MaterialUUID, MaterialUUID); + EXPECT_EQ(next_result.Assets->FindTexture(texture)->Pixels()[2], 90u); + next_lease.Reset(); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.Shutdown()); + } + next_files.Shutdown(); + } }; TEST_F(SceneCookProcessTest, ExecutesRequestedPhase) @@ -1001,6 +1521,15 @@ namespace ZEngine::Cook::Tests } return; } + if (Options.Mode == "cross-version") + { + CheckSharedVersionsAcrossPackages(); + if (!HasFailure()) + { + std::cout << "ZE_COOK_SHARED_VERSIONS_VERIFIED\n"; + } + return; + } if (Reading) { const auto result = ConsumePackage(Options.Mode == "read-changed"); @@ -1016,13 +1545,17 @@ namespace ZEngine::Cook::Tests { EXPECT_TRUE(result.Succeeded()) << static_cast(result.Error()); } + if (!HasFailure() && !Options.Custom && (Options.Mode == "read" || Options.Mode == "read-changed")) + { + CheckRuntimeReaders(); + } if (!HasFailure()) { std::cout << ((Options.Mode == "reject-schema" || Options.Mode == "reject-version") ? "ZE_COOK_SCHEMA_REJECTION_VERIFIED\n" : "ZE_COOK_PACKAGE_VERIFIED\n"); } return; } - const bool create = Options.Mode == "write" || Options.Mode == "write-reverse"; + const bool create = Options.Mode == "write" || Options.Mode == "write-reverse" || Options.Mode == "write-blend" || Options.Mode == "write-hierarchy" || Options.Mode == "write-blend-camera" || Options.Mode == "write-hierarchy-camera" || Options.Mode == "write-switch" || Options.Mode == "write-camera" || Options.Mode == "write-switch-camera" || Options.Mode == "write-switch-no-camera" || Options.Mode == "write-switch-unique-material"; if (Options.Mode == "snapshot-write") { bool ready = false; @@ -1059,7 +1592,7 @@ namespace ZEngine::Cook::Tests std::_Exit(committed ? 72 : 71); }; } - CookFixture(Options.Mode == "force", create || Options.Mode == "changed" || Options.Mode == "rebuild" || Options.Mode == "snapshot-write" || Options.Mode == "schema-change", Options.Mode == "busy"); + CookFixture(Options.Mode == "force" || Options.Mode == "recook", create || Options.Mode == "changed" || Options.Mode == "recook" || Options.Mode == "rebuild" || Options.Mode == "snapshot-write" || Options.Mode == "schema-change", Options.Mode == "busy"); if (!HasFailure() && Options.Mode == "snapshot-write") { WriteText("/output/desktop-vulkan-le64/.ze-test-commit.done", "done"); @@ -1094,13 +1627,18 @@ namespace ZEngine::Cook::Tests } return RunProjectSmoke(std::filesystem::path(argv[2]), std::filesystem::path(argv[3]), budget, std::filesystem::path(argv[1]) == "smoke-stress" ? 65536u : UINT32_MAX); } - const bool custom = argc == 4 && std::filesystem::path(argv[1]) == "custom"; - if (argc != 3 && !custom) + const bool custom = argc == 4 && std::filesystem::path(argv[1]) == "custom"; + const bool cross_version = argc == 4 && std::filesystem::path(argv[1]) == "cross-version"; + if (argc != 3 && !custom && !cross_version) { return 2; } std::filesystem::path mode(argv[custom ? 2 : 1]); - const bool version_two = custom && (mode == "schema-change" || mode == "read-v2" || mode == "repeat-v2" || mode == "force-v2" || mode == "reject-migration"); + if (mode == "cross-version" && !cross_version) + { + return 2; + } + const bool version_two = custom && (mode == "schema-change" || mode == "read-v2" || mode == "repeat-v2" || mode == "force-v2" || mode == "reject-migration"); if (version_two && mode != "schema-change" && mode != "reject-migration") { mode = mode == "read-v2" ? "read" : mode == "repeat-v2" ? "repeat" : "force"; @@ -1110,11 +1648,12 @@ namespace ZEngine::Cook::Tests #if defined(__linux__) disk_full_phase = !custom && (mode == "disk-full" || mode == "disk-full-manifest"); #endif - if (!custom_phase && !disk_full_phase && mode != "stress-fixture" && mode != "write" && mode != "write-reverse" && mode != "repeat" && mode != "force" && mode != "changed" && mode != "read" && mode != "read-changed" && mode != "reject-missing" && mode != "reject-corrupt" && mode != "interrupt-before" && mode != "interrupt-after" && mode != "busy" && mode != "rebuild" && mode != "snapshot-read" && mode != "snapshot-write" && mode != "corrupt-texture") + if (!custom_phase && !disk_full_phase && mode != "cross-version" && mode != "stress-fixture" && mode != "write" && mode != "write-reverse" && mode != "write-blend" && mode != "write-hierarchy" && mode != "write-blend-camera" && mode != "write-hierarchy-camera" && mode != "write-switch" && mode != "write-camera" && mode != "write-switch-camera" && mode != "write-switch-no-camera" && mode != "write-switch-unique-material" && mode != "repeat" && mode != "force" && mode != "recook" && mode != "changed" && mode != "read" && mode != "read-changed" && + mode != "reject-missing" && mode != "reject-corrupt" && mode != "interrupt-before" && mode != "interrupt-after" && mode != "busy" && mode != "rebuild" && mode != "snapshot-read" && mode != "snapshot-write" && mode != "corrupt-texture") { return 2; } - Options = {mode, std::filesystem::path(argv[custom ? 3 : 2]), custom, version_two ? 2u : 1u}; + Options = {mode, std::filesystem::path(argv[custom ? 3 : 2]), custom, version_two ? 2u : 1u, cross_version ? std::filesystem::path(argv[3]) : std::filesystem::path{}}; int test_argc = 1; char name[] = "SceneCookProcess"; char* test_argv[] = {name, nullptr}; diff --git a/ZEngine/tests/ECS/SceneInstance_test.cpp b/ZEngine/tests/ECS/SceneInstance_test.cpp index 207c28a5a..4a6a77c0a 100644 --- a/ZEngine/tests/ECS/SceneInstance_test.cpp +++ b/ZEngine/tests/ECS/SceneInstance_test.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -8,16 +9,48 @@ #include #include #include +#include #include #include #include #include #include +#include "../Cook/CookObjectTestSupport.h" using namespace ZEngine::Core::Memory; using namespace ZEngine::ECS; using namespace ZEngine::ECS::Components; +namespace +{ + namespace VFS = ZEngine::Core::VFS; + + struct CookedSceneFile + { + ZEngine::Cook::Tests::ObjectFile File = {}; + ZEngine::Cook::CookObjectDescriptor Descriptor = {}; + + bool Capture(std::span bytes, IAllocationRoute& route, ZEngine::Managers::CookedObjectSnapshot* output) + { + ZEngine::Core::Hash::SHA256 hash = {}; + ZEngine::Cook::Digest256 digest = {}; + if (hash.Update({bytes.data(), bytes.size()}).Failed() || hash.Finalize(&digest).Failed() || ZEngine::Cook::MakeCookObjectDescriptor(digest, bytes.size(), ZEngine::Cook::CookSceneObjectExtension, &Descriptor).Failed()) + { + return false; + } + File.Bytes = {bytes.data(), bytes.size()}; + ZEngine::Cook::CookObjectReadFns reader = { + this, + [](void* context, const VFS::VFSRelativePath& path) { + auto& source = *static_cast(context); + return path == source.Descriptor.Path ? VFS::VFSResult::Ok(&source.File) : VFS::VFSResult::Fail(VFS::VFSError::NotFound); + }, + [](void*, VFS::IVFSFile* file) { return file->Close(); }}; + return ZEngine::Managers::CookedObjectSnapshot::Capture(reader, Descriptor, ZEngine::Cook::CookSceneObjectExtension, route, bytes.size(), output).Succeeded(); + } + }; +} // namespace + static_assert(!std::is_copy_constructible_v); static_assert(std::is_nothrow_move_constructible_v); static_assert(std::is_nothrow_move_assignable_v); @@ -202,6 +235,52 @@ TEST_F(SceneInstanceTest, RefusedDocumentWorkingStorageLeavesNoRetainedAllocatio vfs.Shutdown(); } +TEST_F(SceneInstanceTest, DecodedCookedDocumentUsesOwnedStorageButCannotPopulateWithoutPackageValidation) +{ + AddSourceEntity(); + SceneDocument source = {}; + ASSERT_TRUE(Capture(&source)); + ZEngine::Core::Containers::Array binary = {}; + ASSERT_TRUE(BinarySceneCodec::Encode(source, Schemas, &Memory.MainArena, &binary, &Diagnostics)); + SlabAllocationRoute route = {}; + ASSERT_TRUE(route.Initialize(&Memory.MainArena, ZMega(1))); + CookedSceneFile file = {}; + ZEngine::Managers::CookedObjectSnapshot snapshot = {}; + ASSERT_TRUE(file.Capture({binary.data(), binary.size()}, route, &snapshot)); + ASSERT_TRUE(Create()); + EXPECT_FALSE(Instance->PopulateCooked(Schemas, &Diagnostics)); + Diagnostics.Clear(); + ASSERT_TRUE(Instance->ReadCooked(snapshot, Schemas, ZMega(8), &Diagnostics)); + snapshot.Reset(); + binary[0] ^= 1; + EXPECT_EQ(Instance->World().AliveCount(), 0u); + EXPECT_FALSE(Instance->TryReleaseDocumentForDiscard(&source)); + EXPECT_FALSE(Instance->PopulateSource(Schemas, &Diagnostics)); + Diagnostics.Clear(); + EXPECT_FALSE(Instance->PopulateCooked(Schemas, &Diagnostics)); + EXPECT_EQ(Instance->World().AliveCount(), 0u); + EXPECT_EQ(Instance->DocumentMemory().Capacity, ZMega(8)); + EXPECT_EQ(Instance->DocumentMemory().Working, 0u); + Instance.reset(); +} + +TEST_F(SceneInstanceTest, InvalidVerifiedCookedBytesDoNotPopulateOrEscapeTheOwner) +{ + std::array invalid = {}; + SlabAllocationRoute route = {}; + ASSERT_TRUE(route.Initialize(&Memory.MainArena, ZMega(1))); + CookedSceneFile file = {}; + ZEngine::Managers::CookedObjectSnapshot snapshot = {}; + ASSERT_TRUE(file.Capture(invalid, route, &snapshot)); + ASSERT_TRUE(Create()); + EXPECT_FALSE(Instance->ReadCooked(snapshot, Schemas, ZMega(8), &Diagnostics)); + EXPECT_EQ(Instance->World().AliveCount(), 0u); + EXPECT_EQ(Instance->Document().Settings(), nullptr); + EXPECT_FALSE(Instance->PopulateCooked(Schemas, &Diagnostics)); + EXPECT_TRUE(Diagnostics.HasErrors()); + Instance.reset(); +} + TEST_F(SceneInstanceTest, LeaseAcquisitionIsBoundedAndCannotReplaceALiveLease) { SceneStorageLease first = {}; diff --git a/ZEngine/tests/ECS/SceneRuntimeCamera_test.cpp b/ZEngine/tests/ECS/SceneRuntimeCamera_test.cpp new file mode 100644 index 000000000..c94fb50e0 --- /dev/null +++ b/ZEngine/tests/ECS/SceneRuntimeCamera_test.cpp @@ -0,0 +1,123 @@ +#include +#include +#include +#include +#include +#include +#include + +using ZEngine::ECS::CaptureSceneRuntimeCamera; +using ZEngine::ECS::Components::CameraComponent; +using ZEngine::ECS::Components::TransformComponent; +using ZEngine::ECS::Components::UUIDComponent; + +namespace ZEngine::ECS +{ + class SceneRuntimeCameraTest : public testing::Test + { + protected: + Core::Memory::MemoryManager Memory = {}; + Scene World = {}; + SceneDiagnostics Diagnostics = {}; + Rendering::Cameras::CameraFrameData Frame = {}; + const uuids::uuid First = *uuids::uuid::from_string("11111111-1111-4111-8111-111111111111"); + const uuids::uuid Second = *uuids::uuid::from_string("22222222-2222-4222-8222-222222222222"); + + void SetUp() override + { + Memory.Initialize(ZMega(32), {}); + World.Initialize(&Memory.MainArena); + } + void TearDown() override + { + World.Shutdown(); + } + EntityID AddCamera(const uuids::uuid& id, bool main) + { + const auto entity = World.CreateEntity(); + World.AddComponent(entity, {id}); + World.AddComponent(entity, {}); + CameraComponent camera = {}; + camera.IsMain = main; + World.AddComponent(entity, camera); + return entity; + } + bool Capture(const uuids::uuid& id = {}, uint32_t width = 1600, uint32_t height = 900) + { + return CaptureSceneRuntimeCamera(World, id, width, height, &Frame, &Diagnostics); + } + }; + + TEST_F(SceneRuntimeCameraTest, RequiresExactlyOneMainUnlessAUuidIsSpecified) + { + const auto first = AddCamera(First, true); + const auto second = AddCamera(Second, true); + World.GetComponent(second)->Position = {1.f, 2.f, 3.f}; + World.SynchronizeHierarchy(); + EXPECT_FALSE(Capture()); + ASSERT_TRUE(Capture(Second)); + EXPECT_FLOAT_EQ(Frame.Position.x, 1.f); + EXPECT_FLOAT_EQ(Frame.Position.y, 2.f); + EXPECT_FLOAT_EQ(Frame.Position.z, 3.f); + EXPECT_FLOAT_EQ(Frame.View(0, 3), -1.f); + EXPECT_FLOAT_EQ(Frame.View(1, 3), -2.f); + EXPECT_FLOAT_EQ(Frame.View(2, 3), -3.f); + EXPECT_LT(Frame.Projection(1, 1), 0.f); + EXPECT_FLOAT_EQ(Frame.Projection(1, 1) / Frame.Projection(0, 0), -1600.f / 900.f); + World.GetComponent(second)->IsMain = false; + ASSERT_TRUE(Capture()); + EXPECT_FLOAT_EQ(Frame.Position.x, 0.f); + World.GetComponent(first)->IsMain = false; + EXPECT_FALSE(Capture()); + EXPECT_FALSE(Capture(*uuids::uuid::from_string("33333333-3333-4333-8333-333333333333"))); + } + + TEST_F(SceneRuntimeCameraTest, UsesSynchronizedParentTransformAndRejectsInvalidInputs) + { + const auto parent = World.CreateEntity(); + TransformComponent parent_transform = {}; + parent_transform.Position = {4.f, 0.f, 0.f}; + World.AddComponent(parent, parent_transform); + const auto camera = AddCamera(First, true); + World.GetComponent(camera)->Position = {1.f, 0.f, 0.f}; + ASSERT_TRUE(World.SetParent(camera, parent)); + EXPECT_FALSE(Capture()); // hierarchy has not been synchronized + World.SynchronizeHierarchy(); + ASSERT_TRUE(Capture()); + EXPECT_FLOAT_EQ(Frame.Position.x, 5.f); + EXPECT_FALSE(Capture({}, 0, 900)); + auto* projection = World.GetComponent(camera); + projection->Near = projection->Far; + EXPECT_FALSE(Capture()); + projection->Near = 0.1f; + projection->FovY = std::numeric_limits::infinity(); + EXPECT_FALSE(Capture()); + projection->FovY = 60.f; + World.GetComponent(camera)->Scale = {0.f, 1.f, 1.f}; + World.SynchronizeHierarchy(); + EXPECT_FALSE(Capture()); + } + + TEST_F(SceneRuntimeCameraTest, AmbiguousOrIncompleteCameraDoesNotOverwriteThePreviousFrame) + { + const auto first = AddCamera(First, true); + World.GetComponent(first)->Position = {2.f, 3.f, 4.f}; + World.SynchronizeHierarchy(); + ASSERT_TRUE(Capture(First)); + const auto original = Frame; + + const auto duplicate = AddCamera(First, false); + World.SynchronizeHierarchy(); + EXPECT_FALSE(Capture(First)); + EXPECT_FLOAT_EQ(Frame.Position.x, original.Position.x); + EXPECT_FLOAT_EQ(Frame.Position.y, original.Position.y); + EXPECT_FLOAT_EQ(Frame.View(0, 3), original.View(0, 3)); + World.DestroyEntity(duplicate); + World.RemoveComponent(first); + World.SynchronizeHierarchy(); + EXPECT_FALSE(Capture(First)); + EXPECT_FLOAT_EQ(Frame.Position.x, original.Position.x); + EXPECT_FLOAT_EQ(Frame.Projection(0, 0), original.Projection(0, 0)); + EXPECT_FALSE(CaptureSceneRuntimeCamera(World, First, 1600, 900, nullptr, &Diagnostics)); + } +} // namespace ZEngine::ECS diff --git a/ZEngine/tests/Misc/ProfilerCapture_test.cpp b/ZEngine/tests/Misc/ProfilerCapture_test.cpp index d18fef4b1..662366545 100644 --- a/ZEngine/tests/Misc/ProfilerCapture_test.cpp +++ b/ZEngine/tests/Misc/ProfilerCapture_test.cpp @@ -119,14 +119,17 @@ TEST(ProfilerCaptureTest, TimedCaptureStopsBeforeLaterSamplesWithoutDebuggerInte ProfilerBuffer::StopCapture(); EXPECT_FALSE(ProfilerBuffer::StartTimedCapture(0)); ASSERT_TRUE(ProfilerBuffer::StartTimedCapture(100, "Timed.")); + EXPECT_FALSE(ProfilerBuffer::CaptureDeadlinePassed()); ProfilerBuffer::RecordValue("Timed.Marker", 1); EXPECT_EQ(ProfilerBuffer::CaptureCount(), 1u); EXPECT_FALSE(ProfilerBuffer::StartTimedCapture(100, "Other.")); std::this_thread::sleep_for(std::chrono::milliseconds(120)); + EXPECT_TRUE(ProfilerBuffer::CaptureDeadlinePassed()); ProfilerBuffer::Record("Timed.TooLate", 10); EXPECT_EQ(ProfilerBuffer::CaptureCount(), 1u); EXPECT_EQ(ProfilerBuffer::CaptureDropped(), 0u); ASSERT_TRUE(ProfilerBuffer::StartCapture("Fresh.")); + EXPECT_FALSE(ProfilerBuffer::CaptureDeadlinePassed()); ProfilerBuffer::Record("Fresh.NoOldDeadline", 10); EXPECT_EQ(ProfilerBuffer::CaptureCount(), 1u); ProfilerBuffer::StopCapture(); diff --git a/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-candidate.json b/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-candidate.json new file mode 100644 index 000000000..7a45c4d49 --- /dev/null +++ b/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-candidate.json @@ -0,0 +1,31 @@ +{ + "status": "user-reviewed candidate; not full visual qualification", + "review": "User confirmed character and lighting looked correct enough for a reference candidate in this session, 2026-11-17.", + "backend": "macOS Vulkan on Apple M4 Pro (MoltenVK)", + "capture_space": "RGB8 UNORM; scene target readback", + "uncompressed_ppm_sha256": "9edd0f04f710bebbc001d7301bfee520eb3b4ae2c0e7486159f9979838332e73", + "compressed_sha256": "2bb1881ec70d70dc0afd9d373d45380c8f3d24cfcff1687507e1137d705236a8", + "scene_sha256": "da6d6c6050200a0d31b17e05ef8b53950ef0ddc705ae4fe7f4e4dfacece9cd31", + "augmented_scene_sha256": "0554a318872db8a0c86f03c65bb45cf9cac65e8b7539b0a68dddc7399411a66a", + "cooked_manifest_sha256": "3dc692d4affcb702f2255c48f68c34e6652bc9b5c5590efb85c5b735a53c43da", + "camera_uuid": "b1b1b1b1-7171-4141-8181-b1b1b1b1b1b1", + "camera_position": [ + 0.5, + 2, + 6 + ], + "viewport": [ + 2560, + 1378 + ], + "character_region": [ + 1150, + 450, + 220, + 450 + ], + "limitations": [ + "Camera is added to a copy of level01.", + "The user reviewed character and lighting only as a candidate; transparency, materials, hierarchy, replacement and backend-specific repeatability remain to be qualified." + ] +} diff --git a/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-candidate.ppm.gz b/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-candidate.ppm.gz new file mode 100644 index 000000000..a69facc87 Binary files /dev/null and b/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-candidate.ppm.gz differ diff --git a/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-fixed-candidate.json b/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-fixed-candidate.json new file mode 100644 index 000000000..bb347419a --- /dev/null +++ b/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-fixed-candidate.json @@ -0,0 +1,69 @@ +{ + "status": "user-approved macOS fixed-viewport visual reference; not full M2 qualification", + "backend": "macOS Vulkan on Apple M4 Pro (MoltenVK)", + "capture_space": "RGB8 UNORM scene-target readback", + "viewport": [ + 1280, + 720 + ], + "character_region": [ + 560, + 225, + 160, + 250 + ], + "ppm_sha256": "c8e487774ecf7e42cc9b87a6b7ae771488c77a98c15da0d4f4368f388a5c95ed", + "compressed_sha256": "b9c5ca700feeead1a976c7dfd91b8a72c504b2502868014dae9085d665565e6e", + "pixel_sha256": "dad5a00f088726f6c02546cd2084172197a1c3db67c6c0703565d4b726872810", + "scene_sha256": "da6d6c6050200a0d31b17e05ef8b53950ef0ddc705ae4fe7f4e4dfacece9cd31", + "augmented_scene_sha256": "0554a318872db8a0c86f03c65bb45cf9cac65e8b7539b0a68dddc7399411a66a", + "manifest_sha256": "3dc692d4affcb702f2255c48f68c34e6652bc9b5c5590efb85c5b735a53c43da", + "camera_uuid": "b1b1b1b1-7171-4141-8181-b1b1b1b1b1b1", + "source_vs_cooked_changed_pixels": 0, + "repeat_changed_pixels": 0, + "limitations": [ + "Runtime binary and shader identities were not recorded for this capture.", + "Approval of this image does not qualify every visual semantic or the overall M2 profile." + ], + "preview_png_sha256": "4da68f038761e2b7c9e8cf8d10c66fa9581d61f42617cefd2666ccf16785cd01", + "matching_later_capture": { + "binary_sha256": { + "cooker_sha256": "cc4bc8c155a47c8a021030132134589bd4f7d68fc74f1568e69ff22c4294ba1f", + "runtime_sha256": "07cd943dde32946864f82bfc935a7e9baa142c6b21d130369b806123c366352b" + }, + "shader_cache": { + "sha256": "dfd8d94eb2a7e0afc9e34f4644052fb2d86de3e7396cb0ff5496d89717d21916", + "files": 44 + }, + "device": { + "gpu": "Apple M4 Pro", + "driver_path": "/usr/local/lib/libMoltenVK.dylib", + "driver_sha256": "0b398933adeb1aba84aff76ef438f54f17e7deb6960beb5711fa6e23cd192cdb" + }, + "manifest_sha256": "3dc692d4affcb702f2255c48f68c34e6652bc9b5c5590efb85c5b735a53c43da", + "pixel_sha256": "dad5a00f088726f6c02546cd2084172197a1c3db67c6c0703565d4b726872810", + "source_cooked_changed_pixels": 0, + "candidate_changed_pixels": 0, + "same_build_repeat_changed_pixels": 0, + "same_build_repeat_maximum_channel_delta": 0 + }, + "matching_corrected_domain_capture": { + "binary_sha256": { + "cooker_sha256": "cc4bc8c155a47c8a021030132134589bd4f7d68fc74f1568e69ff22c4294ba1f", + "runtime_sha256": "14af15626c422b8d299fc2483c7a650067fc65028d510565289f0122a6533b5e" + }, + "shader_cache": { + "sha256": "dfd8d94eb2a7e0afc9e34f4644052fb2d86de3e7396cb0ff5496d89717d21916", + "files": 44 + }, + "device": { + "gpu": "Apple M4 Pro", + "driver_path": "/usr/local/lib/libMoltenVK.dylib", + "driver_sha256": "0b398933adeb1aba84aff76ef438f54f17e7deb6960beb5711fa6e23cd192cdb" + }, + "manifest_sha256": "3dc692d4affcb702f2255c48f68c34e6652bc9b5c5590efb85c5b735a53c43da", + "runtime_pixel_sha256": "dad5a00f088726f6c02546cd2084172197a1c3db67c6c0703565d4b726872810", + "source_cooked_changed_pixels": 0, + "candidate_changed_pixels": 0 + } +} diff --git a/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-fixed-candidate.png b/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-fixed-candidate.png new file mode 100644 index 000000000..e58c0a902 Binary files /dev/null and b/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-fixed-candidate.png differ diff --git a/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-fixed-candidate.ppm.gz b/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-fixed-candidate.ppm.gz new file mode 100644 index 000000000..d57540153 Binary files /dev/null and b/ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-fixed-candidate.ppm.gz differ diff --git a/ZEngine/tests/Rendering/GeometryStreamingManagerTest.cpp b/ZEngine/tests/Rendering/GeometryStreamingManagerTest.cpp index 84dc0da3c..2415b6bb1 100644 --- a/ZEngine/tests/Rendering/GeometryStreamingManagerTest.cpp +++ b/ZEngine/tests/Rendering/GeometryStreamingManagerTest.cpp @@ -43,6 +43,11 @@ struct RRMTestHelper return renderer.m_sky_environment; } + static bool SubmitCookedSky(ZEngine::Rendering::Renderers::GraphicRenderer& renderer, const ZEngine::Rendering::Scenes::SkyConfig& sky, const RenderResourceManager::SceneEnvironmentPreparation& preparation) + { + return renderer.SubmitSkySnapshot(sky, {}, 1, ZEngine::Rendering::Scenes::SkyEnvironmentPublication::Immediate, &preparation); + } + static void InitializeSky(ZEngine::Rendering::Renderers::GraphicRenderer& renderer, ZEngine::Hardwares::VulkanDevice* device, ZEngine::Tests::ScenePipelinePreparationTestHost& pipelines, ZEngine::Rendering::Textures::TextureHandle source, const ZEngine::Rendering::EnvironmentLightingResources& lighting) { renderer.Device = device; @@ -72,6 +77,11 @@ struct RRMTestHelper resources.m_scene_resources_sequence = UINT64_MAX - 1; } + static void MarkSceneCooked(RenderResourceManager& resources, RenderResourceManager::SceneResourcePreparationId id) + { + resources.m_scene_resources[id.Slot].Cooked = true; + } + static void BeginSubmittedSceneSky(ZEngine::Rendering::Renderers::GraphicRenderer& renderer, RenderResourceManager& resources) { RenderResourceManager::SceneEnvironmentPreparation preparation = {}; @@ -255,9 +265,9 @@ struct RRMTestHelper return index == UINT32_MAX ? 0 : rrm.m_pinned_mesh_geometry[index].PinCount; } - static void TickPreparation(RenderResourceManager& rrm, uint64_t completed = 0, bool failed = false) + static void TickPreparation(RenderResourceManager& rrm, uint64_t completed = 0, bool failed = false, bool invalidate_ready = true) { - rrm.TickSceneResourcePreparations(completed, failed); + rrm.TickSceneResourcePreparations(completed, failed, invalidate_ready); } static void SetUploadProgress(RenderResourceManager& rrm, uint64_t recorded, uint64_t accepted, bool open) @@ -1345,6 +1355,21 @@ TEST_F(SceneEnvironmentHandoffTest, CandidateLeavesOutgoingSkyUnchangedUntilItsM EXPECT_TRUE(Result(id).FrameRecorded); } +TEST_F(SceneEnvironmentHandoffTest, RetainedCookedHDRISourceDoesNotRequeueAsMissingAfterActivation) +{ + auto input = Sphere(); + input.Sky.Mode = Scenes::SkyMode::HDRI; + input.Sky.EnvironmentMap = *uuids::uuid::from_string("60000000-0000-0000-0000-000000000006"); + RRM::SceneEnvironmentPreparation preparation = {}; + preparation.Input = input; + preparation.CookedSource = {20, 1}; + preparation.CookedSourceHash = 1234; + ASSERT_TRUE(RRMTestHelper::SubmitCookedSky(*Renderer, input.Sky, preparation)); + // Submitting the same verified identity must not look like a transition to + // an unavailable source merely because the authoring registry is empty. + EXPECT_FALSE(RRMTestHelper::SubmitCookedSky(*Renderer, input.Sky, preparation)); +} + TEST_F(SceneEnvironmentHandoffTest, CancellationBeforePublicationReleasesTheClaimAndOutgoingEditsCanResume) { const auto outgoing = Sphere(1.0f, 500); @@ -1366,6 +1391,30 @@ TEST_F(SceneEnvironmentHandoffTest, CancellationBeforePublicationReleasesTheClai EXPECT_FALSE(Renderer->ApplySkyConfig({}, {}, 2, 2, id)); } +TEST_F(SceneEnvironmentHandoffTest, CancelledCandidateDoesNotResubmitTheActiveCookedSky) +{ + const auto active = Sphere(1.0f, 7); + const auto active_id = Queue(active); + RRMTestHelper::MarkSceneCooked(Fixture->rrm, active_id); + Renderer->PollSceneEnvironmentPreparations(); + ASSERT_EQ(Result(active_id).Environment, RRM::SceneEnvironmentPreparationState::Ready); + auto active_frame = Fixture->rrm.AcquireSceneResourceLease(active_id, 2); + ASSERT_TRUE(active_frame); + ASSERT_TRUE(Renderer->ApplySkyConfig(active.Sky, {}, active.SkyRevision, 2, active_id)); + RRMTestHelper::FinishSkyFrame(*Renderer, true); + const auto candidate = Sphere(4.0f, 8); + const auto candidate_id = Queue(candidate, 2, 2); + Renderer->PollSceneEnvironmentPreparations(); + ASSERT_EQ(Result(candidate_id).Environment, RRM::SceneEnvironmentPreparationState::Ready); + ASSERT_TRUE(Fixture->rrm.ReleaseSceneResourcePreparation(candidate_id)); + RRMTestHelper::TickPreparation(Fixture->rrm); + Renderer->PollSceneEnvironmentPreparations(); + const auto latest_revision = RRMTestHelper::Sky(*Renderer).GetLatestRevision(); + ASSERT_TRUE(Renderer->ApplySkyConfig(active.Sky, {}, active.SkyRevision, 2, active_id)); + EXPECT_EQ(RRMTestHelper::Sky(*Renderer).GetLatestRevision(), latest_revision); + EXPECT_EQ(RRMTestHelper::Sky(*Renderer).GetPresentationConfig(), active.Sky); +} + TEST_F(SceneEnvironmentHandoffTest, MailboxRetentionPreventsTheNextCandidateFromSupersedingTheFirstSky) { const auto first = Sphere(3.0f); @@ -2041,9 +2090,9 @@ TEST_F(SceneResourceBundleTest, CancellationDuringPreparationWaitsForAcceptedPro RRMTestHelper::TickPreparation(rrm, 100); // A signal alone cannot accept the open batch. EXPECT_EQ(RRMTestHelper::TextureReferences(rrm), 2u); RRMTestHelper::SetUploadProgress(rrm, 4, 4, false); - RRMTestHelper::TickPreparation(rrm, 3); + RRMTestHelper::TickPreparation(rrm, 3, true, false); // A later rejected batch cannot bypass an accepted upload. EXPECT_EQ(RRMTestHelper::TextureReferences(rrm), 2u); - RRMTestHelper::TickPreparation(rrm, 4); + RRMTestHelper::TickPreparation(rrm, 4, true, false); EXPECT_EQ(RRMTestHelper::TextureReferences(rrm), 0u); RRM::SceneResourcePreparationResult result = {}; EXPECT_FALSE(rrm.PollSceneResourcePreparation(id, &result)); diff --git a/ZEngine/tests/Rendering/GpuAllocatorTest.cpp b/ZEngine/tests/Rendering/GpuAllocatorTest.cpp index 10b0ed353..e07beb7a5 100644 --- a/ZEngine/tests/Rendering/GpuAllocatorTest.cpp +++ b/ZEngine/tests/Rendering/GpuAllocatorTest.cpp @@ -1,16 +1,24 @@ #include #include +#include #include #include +#include #include #include +#include #include #include #include #include #include #include +#include #include +#include +#include +#include +#include #include #include #include @@ -18,9 +26,14 @@ #include #include #include +#include +#include +#include +#include #include #include #include +#include #include #include #include @@ -31,6 +44,9 @@ #include #include #include +#include +#include "../Cook/CookTestAllocator.h" +#include "CookManifestGolden.h" #include "GizmoShaderSources.h" #include "GridLabelShaderSources.h" @@ -1306,12 +1322,14 @@ TEST_F(GpuAllocatorTest, ByteAdmissionChargesRequirementsBeforeBackingAndReturns EXPECT_EQ(LiveAllocationBytes(), baseline); EXPECT_EQ(local->GetAllocationStatistics().ReservedRecords, 0u); EXPECT_EQ(local->GetAllocationStatistics().AdmittedBytes, 0u); + EXPECT_EQ(local->GetAllocationStatistics().DomainBudgetRefusals[domain_index], 1u); EXPECT_EQ(queue->GetStatistics().Reserved, 0u); limits.TotalBytes = ZMega(1); limits.DomainBytes[domain_index] = 0; ASSERT_TRUE(local->SetAllocationLimits(limits)); EXPECT_FALSE(local->TryAllocateBuffer(64, usage, domain, nullptr, &status)); EXPECT_EQ(status, VK_ERROR_OUT_OF_DEVICE_MEMORY); + EXPECT_EQ(local->GetAllocationStatistics().DomainBudgetRefusals[domain_index], 2u); limits.DomainBytes[domain_index] = ZMega(1); ASSERT_TRUE(local->SetAllocationLimits(limits)); auto buffer = local->TryAllocateBuffer(64, usage, domain); @@ -1319,8 +1337,9 @@ TEST_F(GpuAllocatorTest, ByteAdmissionChargesRequirementsBeforeBackingAndReturns VkMemoryRequirements requirements = {}; vkGetBufferMemoryRequirements(s_vk->Device, buffer.Handle, &requirements); const auto live = local->GetAllocationStatistics(); - EXPECT_EQ(live.AdmittedBytes, requirements.size); - EXPECT_EQ(live.DomainAdmittedBytes[domain_index], requirements.size); + EXPECT_GE(live.AdmittedBytes, requirements.size); + EXPECT_EQ(live.DomainAdmittedBytes[domain_index], live.AdmittedBytes); + EXPECT_EQ(live.DomainPeakAdmittedBytes[domain_index], live.DomainAdmittedBytes[domain_index]); EXPECT_LE(live.AllocationBytes, live.AdmittedBytes); limits.TotalBytes = live.AdmittedBytes - 1; EXPECT_FALSE(local->SetAllocationLimits(limits)); @@ -1333,6 +1352,7 @@ TEST_F(GpuAllocatorTest, ByteAdmissionChargesRequirementsBeforeBackingAndReturns EXPECT_EQ(queue->GetStatistics().Reserved, 2u); EXPECT_FALSE(local->TryAllocateBuffer(1, usage, domain, nullptr, &status)); EXPECT_EQ(status, VK_ERROR_OUT_OF_DEVICE_MEMORY); + EXPECT_EQ(local->GetAllocationStatistics().DomainBudgetRefusals[domain_index], 3u); EXPECT_EQ(local->GetAllocationStatistics().ReservedRecords, 1u); EXPECT_EQ(queue->GetStatistics().Reserved, 2u); local->FreeBuffer(buffer); @@ -1340,10 +1360,34 @@ TEST_F(GpuAllocatorTest, ByteAdmissionChargesRequirementsBeforeBackingAndReturns local->FreeBuffer(alias); EXPECT_EQ(local->GetAllocationStatistics().ReservedRecords, 0u); EXPECT_EQ(local->GetAllocationStatistics().AdmittedBytes, 0u); + EXPECT_EQ(local->GetAllocationStatistics().DomainPeakAdmittedBytes[domain_index], live.AdmittedBytes); EXPECT_EQ(queue->GetStatistics().Reserved, 0u); EXPECT_EQ(LiveAllocationBytes(), baseline); } +TEST_F(GpuAllocatorTest, UnalignedBufferBackingIsAdmittedAndRetiredWithoutExceedingTheLedger) +{ + auto local = std::make_unique(); + local->Allocator = allocator().Allocator; + constexpr VkDeviceSize size = 64 * 1024 + 17; + auto buffer = local->TryAllocateBuffer(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, GpuMemoryDomain::HostStaging); + ASSERT_TRUE(buffer); + const auto live = local->GetAllocationStatistics(); + EXPECT_GE(live.AdmittedBytes, local->GetAllocationSize(buffer.Allocation)); + EXPECT_GE(live.AdmittedBytes, size); + local->FreeBuffer(buffer); + auto limits = local->GetAllocationLimits(); + limits.DomainBytes[static_cast(GpuMemoryDomain::HostStaging)] = live.AdmittedBytes - 1; + ASSERT_TRUE(local->SetAllocationLimits(limits)); + VkResult status = VK_SUCCESS; + EXPECT_FALSE(local->TryAllocateBuffer(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, GpuMemoryDomain::HostStaging, nullptr, &status)); + EXPECT_EQ(status, VK_ERROR_OUT_OF_DEVICE_MEMORY); + const auto after = local->GetAllocationStatistics(); + EXPECT_EQ(after.AdmittedBytes, 0u); + EXPECT_EQ(after.ReservedRecords, 0u); + EXPECT_EQ(after.AllocationCount, 0u); +} + TEST_F(GpuAllocatorTest, RetirementSaturationRefusesNativeCreationAndAliasesThenRecovers) { auto queue = std::make_unique(); @@ -1752,6 +1796,31 @@ struct RRMUploadBatchTestHelper manager.m_streaming_upload_tickets.init(&arena, 8); manager.PublishStreamingUploadTicket(ticket); } + static void MarkTextureSubmitted(RRM& manager, ZEngine::Rendering::Textures::TextureHandle texture) + { + EXPECT_TRUE(manager.TrackTextureDecode(texture)); + manager.MarkTextureUploadSubmitted(texture); + } + static void AcknowledgeGraphAcquires(ZEngine::Rendering::Renderers::RenderGraph& graph, ZEngine::Rendering::Primitives::Semaphore* timeline, uint64_t value) + { + graph.AcknowledgeStreamingAcquires(0, static_cast(graph.SortedPassIndices.size()), timeline, value); + } + static void SetMeshSliceTestSource(ZEngine::Rendering::Meshes::CookedMeshStaging& staging, const ZEngine::Importers::AssetMesh& mesh) + { + // Source capture and its real closure lease are tested separately in a + // cooker process; this fixture supplies a large immutable stream to the + // GPU staging path without constructing another package. + staging.m_mesh = &mesh; + staging.m_vertex_bytes = mesh.Vertices.size() * sizeof(float); + staging.m_index_bytes = mesh.Indices.size() * sizeof(uint32_t); + staging.m_identity = {ZEngine::Cook::CookAssetKind::Mesh, *uuids::uuid::from_string("00000000-0000-4000-8000-000000000042")}; + staging.m_digest.Bytes[0] = 42; + staging.m_phase = ZEngine::Rendering::Meshes::CookedMeshStaging::Phase::SourceReady; + } + static const BufferView& MeshStagingBuffer(const ZEngine::Rendering::Meshes::CookedMeshStaging& staging) + { + return staging.m_staging.m_buffer; + } static bool SceneTextureReady(RRM& manager, const RRM::MaterialVersionPin& material, bool* failed) { return manager.SceneTextureProducersReady(material, failed); @@ -1786,6 +1855,10 @@ struct RRMUploadBatchTestHelper { return manager.PrepareMaterialDescriptorVersions(pin); } + static ZEngine::Rendering::ISceneTextureDescriptorPreparation* SetTextureDescriptorPreparation(RRM& manager, ZEngine::Rendering::ISceneTextureDescriptorPreparation* replacement) + { + return std::exchange(manager.m_scene_descriptor_preparation, replacement); + } static ZEngine::Rendering::SceneTextureDescriptorState PrepareFrameMaterialDescriptors(RRM& manager, uint32_t index) { return manager.PrepareMaterialDescriptorVersions(manager.m_frame_materials[index].Pin); @@ -1795,6 +1868,80 @@ struct RRMUploadBatchTestHelper manager.m_pool.Initialize(&arena, 64, 64, 32); } + static void InitializeCookedGeometry(RRM& manager, ArenaAllocator& arena, BufferView vertex, BufferView index) + { + manager.m_pool.Initialize(&arena, vertex.Size, index.Size, 32); + manager.m_pool.VertexBuffer = vertex; + manager.m_pool.IndexBuffer = index; + } + + static void ReleaseCookedGeometryAfterIdle(RRM& manager) + { + manager.ReleaseCookedMeshUploadsAfterIdle(); + manager.ResetGeometryBuffersInternal(); + manager.m_pool.VertexBuffer = {}; + manager.m_pool.IndexBuffer = {}; + } + + static uint32_t OutstandingCookedMeshUploads(const RRM& manager) + { + uint32_t count = 0; + for (const auto& upload : manager.m_cooked_mesh_uploads) + { + count += upload.State != RRM::CookedMeshUploadState::Invalid; + } + return count; + } + + // Model a batch rejected before queue acceptance, after recording is + // complete. No GPU command from this batch is submitted. + static bool RejectRecordedCookedBatch(RRM& manager) + { + if (!manager.m_batch_mode || !manager.m_batch_cmd) + return false; + if (vkEndCommandBuffer(manager.m_batch_cmd->GetHandle()) != VK_SUCCESS) + return false; + manager.m_batch_cmd->SetState(ZEngine::Hardwares::CommandBufferState::Executable); + manager.m_batch_mode = false; + manager.m_batch_cmd = nullptr; + manager.m_batch_submission_failed = true; + manager.DiscardUnsubmittedBatchStaging(); + manager.PollCookedMeshUploads(); + return true; + } + + static void SetInjectedBatchRejection(RRM& manager) + { + manager.m_batch_submission_failed = true; + } + static bool BatchSubmissionFailed(const RRM& manager) + { + return manager.m_batch_submission_failed; + } + static uint32_t BatchStagingCount(const RRM& manager, uint32_t frame) + { + return manager.m_batch_frames[frame].StagingCount; + } + static VkBuffer BatchStagingHandle(const RRM& manager, uint32_t frame, uint32_t index) + { + return manager.m_batch_frames[frame].StagingBuffers[index].Handle; + } + static uint64_t BatchLastSignal(const RRM& manager, uint32_t frame) + { + return manager.m_batch_frames[frame].LastSignal; + } + static void AppendBatchStaging(RRM& manager, uint32_t frame, BufferView buffer) + { + auto& state = manager.m_batch_frames[frame]; + state.StagingBuffers[state.StagingCount++] = buffer; + } + static void DiscardStagingTail(RRM& manager, uint8_t frame, uint32_t base) + { + manager.m_batch_frame_index = frame; + manager.m_batch_staging_base_count = base; + manager.DiscardUnsubmittedBatchStaging(); + } + static void InitializeSceneResidency(RRM& manager, ArenaAllocator& arena, BufferView vertex, BufferView index, ZEngine::Core::VFS::AssetRegistry* registry) { InitializeGeometry(manager, arena); @@ -4323,20 +4470,222 @@ class RenderResourceManagerBatchTest : public GpuAllocatorTest EXPECT_EQ(vmaCopyAllocationToMemory(allocator().Allocator, Target.Allocation, offset, &value, sizeof(value)), VK_SUCCESS); return value; } - ZEngine::Rendering::Textures::TextureHandle CreateProducerTestTexture(cstring name) + void CheckTextureReadback(ZEngine::Rendering::Textures::TextureHandle texture, std::span expected) { - Device->MaxGlobalTexture = 8; - Device->GlobalTextures.Initialize(&Memory.MainArena, 8); - Device->ImageBufferManager.Initialize(&Memory.MainArena, 8); + const auto* resource = Device->GlobalTextures.Access(texture); + ASSERT_NE(resource, nullptr); + const auto image = Device->ImageBufferManager.Access(resource->BufferHandle)->GetHandle(); + const auto command = Commands[1]->GetHandle(); + VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + ASSERT_EQ(vkBeginCommandBuffer(command, &begin), VK_SUCCESS); + VkImageMemoryBarrier barrier = { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .srcAccessMask = VK_ACCESS_SHADER_READ_BIT, .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = image, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1} + }; + vkCmdPipelineBarrier(command, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + const VkBufferImageCopy copy = { + .imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, + .imageExtent = {1, 1, 1} + }; + vkCmdCopyImageToBuffer(command, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, Target.Handle, 1, ©); + barrier.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + barrier.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + VkBufferMemoryBarrier host_barrier = {.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, .dstAccessMask = VK_ACCESS_HOST_READ_BIT, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .buffer = Target.Handle, .offset = 0, .size = expected.size()}; + vkCmdPipelineBarrier(command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 0, nullptr, 1, &host_barrier, 1, &barrier); + ASSERT_EQ(vkEndCommandBuffer(command), VK_SUCCESS); + const uint64_t value = 1; + const auto timeline = Timeline->GetHandle(); + VkTimelineSemaphoreSubmitInfo timeline_info = {.sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO, .signalSemaphoreValueCount = 1, .pSignalSemaphoreValues = &value}; + VkSubmitInfo submit = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .pNext = &timeline_info, .commandBufferCount = 1, .pCommandBuffers = &command, .signalSemaphoreCount = 1, .pSignalSemaphores = &timeline}; + ASSERT_EQ(vkQueueSubmit(Device->GetQueue(ZEngine::Rendering::QueueType::GRAPHIC_QUEUE).Handle, 1, &submit, VK_NULL_HANDLE), VK_SUCCESS); + Timeline->Wait(value); + uint8_t result[4] = {}; + ASSERT_EQ(vmaCopyAllocationToMemory(allocator().Allocator, Target.Allocation, 0, result, sizeof(result)), VK_SUCCESS); + EXPECT_TRUE(std::equal(expected.begin(), expected.end(), std::begin(result))); + } + + ZEngine::Rendering::Textures::TextureHandle CreateProducerTestTexture(cstring name, bool transfer_source = false, uint32_t capacity = 8) + { + Device->MaxGlobalTexture = capacity; + Device->GlobalTextures.Initialize(&Memory.MainArena, capacity); + Device->ImageBufferManager.Initialize(&Memory.MainArena, capacity); Device->ImageViewCacheSlab.Init(&Memory.MainArena, 32 * 1024); Device->RRM = Manager.get(); ResourceTexturesInitialized = true; ZEngine::Rendering::Specifications::TextureSpecification specification = {}; specification.Width = specification.Height = 1; specification.Format = ZEngine::Rendering::Specifications::ImageFormat::R8G8B8A8_UNORM; + specification.IsUsageTransferSource = transfer_source; return Device->CreateTexture(specification, name); } + void CheckTextureConsumerAcquire(bool cancel, bool wrong_family = false) + { + using DescriptorState = ZEngine::Rendering::SceneTextureDescriptorState; + using Graph = ZEngine::Rendering::Renderers::RenderGraph; + const auto texture = CreateProducerTestTexture("texture-acquire-proof"); + ASSERT_TRUE(texture.Valid()); + RRMUploadBatchTestHelper::InitializeTextureUploadSlots(*Manager, *Device, UploadCommands, false); + TextureUploadSlotsInitialized = true; + uint8_t pixels[4] = {17, 53, 127, 255}; + ASSERT_TRUE(Manager->UploadTextureBuffer(0, 0, texture, pixels, sizeof(pixels)).Valid()); + Manager->SubmitAsyncUploads(); + const auto* upload_ticket = Manager->FindStreamingUploadTicket(texture); + ASSERT_NE(upload_ticket, nullptr); + const auto ticket = *upload_ticket; + ticket.CompletionTimeline->Wait(ticket.CompletionValue); + RRMUploadBatchTestHelper::RetireTextureUploadSlots(*Manager); + auto pin = Manager->PinTextureVersion(texture); + ASSERT_TRUE(pin); + EXPECT_EQ(Manager->PollSubmittedTextureAcquire(pin), DescriptorState::Pending); + + Graph graph = {}; + graph.Device = Device.get(); + graph.Passes.init(&Memory.MainArena, 1); + auto& pass = graph.Passes.push_use({}); + pass.Queue = ZEngine::Rendering::QueueType::GRAPHIC_QUEUE; + pass.StreamingAcquirePlans.init(&Memory.MainArena, 1); + pass.StreamingAcquirePlans.push({.Ticket = ticket}); + graph.SortedPassIndices.init(&Memory.MainArena, 1); + graph.SortedPassIndices.push(0); + RRMUploadBatchTestHelper::AcknowledgeGraphAcquires(graph, Timeline.get(), 1); + EXPECT_NE(Manager->FindStreamingUploadTicket(texture), nullptr); // No barrier recorded. + auto stale = ticket; + ++stale.CompletionValue; + EXPECT_FALSE(Manager->AcknowledgeStreamingUploadAcquire(stale, Timeline.get(), 1, Device->GraphicFamilyIndex)); + stale = ticket; + stale.PostReleaseLayout = VK_IMAGE_LAYOUT_GENERAL; + EXPECT_FALSE(Manager->AcknowledgeStreamingUploadAcquire(stale, Timeline.get(), 1, Device->GraphicFamilyIndex)); + EXPECT_FALSE(Manager->AcknowledgeStreamingUploadAcquire(ticket, nullptr, 1, Device->GraphicFamilyIndex)); + EXPECT_FALSE(Manager->AcknowledgeStreamingUploadAcquire(ticket, Timeline.get(), 0, Device->GraphicFamilyIndex)); + + const auto* resource = Device->GlobalTextures.Access(texture); + ASSERT_NE(resource, nullptr); + const auto image = Device->ImageBufferManager.Access(resource->BufferHandle)->GetHandle(); + const auto command = Commands[1]->GetHandle(); + VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + ASSERT_EQ(vkBeginCommandBuffer(command, &begin), VK_SUCCESS); + VkImageMemoryBarrier2 barrier = { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_NONE, .dstStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT, .dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = image, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1} + }; + VkDependencyInfo dependency = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &barrier}; + vkCmdPipelineBarrier2(command, &dependency); + ASSERT_EQ(vkEndCommandBuffer(command), VK_SUCCESS); + auto gate = std::make_unique(Device.get(), true); + struct GateGuard + { + ZEngine::Rendering::Primitives::Semaphore* Gate = nullptr; + bool Released = false; + void Release() + { + if (Gate && !Released) + { + Gate->Signal(1); + Released = true; + } + } + ~GateGuard() + { + Release(); + } + } gate_guard{gate.get()}; + const auto wait = gate->GetHandle(); + const auto signal = Timeline->GetHandle(); + const uint64_t value = 1; + const VkPipelineStageFlags wait_stage = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + VkTimelineSemaphoreSubmitInfo timeline_info = {.sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO, .waitSemaphoreValueCount = 1, .pWaitSemaphoreValues = &value, .signalSemaphoreValueCount = 1, .pSignalSemaphoreValues = &value}; + VkSubmitInfo submit = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .pNext = &timeline_info, .waitSemaphoreCount = 1, .pWaitSemaphores = &wait, .pWaitDstStageMask = &wait_stage, .commandBufferCount = 1, .pCommandBuffers = &command, .signalSemaphoreCount = 1, .pSignalSemaphores = &signal}; + ASSERT_EQ(vkQueueSubmit(Device->GetQueue(ZEngine::Rendering::QueueType::GRAPHIC_QUEUE).Handle, 1, &submit, VK_NULL_HANDLE), VK_SUCCESS); + pass.StreamingAcquirePlans[0].BarrierRecorded = true; + if (wrong_family) + { + // Inject a mismatched receipt; this fixture has only one queue family. + EXPECT_TRUE(Manager->AcknowledgeStreamingUploadAcquire(ticket, Timeline.get(), value, Device->GraphicFamilyIndex + 1)); + } + else + { + RRMUploadBatchTestHelper::AcknowledgeGraphAcquires(graph, Timeline.get(), value); + EXPECT_TRUE(pass.StreamingAcquirePlans[0].Acknowledged); + } + EXPECT_EQ(Manager->FindStreamingUploadTicket(texture), nullptr); + EXPECT_FALSE(Manager->AcknowledgeStreamingUploadAcquire(ticket, Timeline.get(), value, Device->GraphicFamilyIndex)); + EXPECT_EQ(Manager->PollSubmittedTextureAcquire(pin), DescriptorState::Pending); + EXPECT_FALSE(Manager->ForgetResourceTimeline(Timeline.get())); + if (cancel) + { + Manager->ReleaseTexture(texture); + pin.Reset(); + RRMUploadBatchTestHelper::FlushTextureReleases(*Manager); + EXPECT_TRUE(Device->GlobalTextures.IsLive(texture)); + EXPECT_TRUE(Device->TextureHandleToDispose.empty()); + EXPECT_FALSE(Manager->CanWriteTextureVersion(texture)); + } + gate_guard.Release(); + Timeline->Wait(value); + EXPECT_TRUE(Manager->ForgetResourceTimeline(Timeline.get())); + if (cancel) + { + RRMUploadBatchTestHelper::FlushTextureReleases(*Manager); + ZEngine::Hardwares::TextureDisposeEntry entry = {}; + EXPECT_TRUE(Device->TextureHandleToDispose.pop(entry)); + EXPECT_EQ(entry.Handle, texture); + } + else if (wrong_family) + { + EXPECT_EQ(Manager->PollSubmittedTextureAcquire(pin), DescriptorState::Failed); + pin.Reset(); + EXPECT_FALSE(Manager->BeginTextureVersionWrite(texture)); + } + else + { + EXPECT_EQ(Manager->PollSubmittedTextureAcquire(pin), DescriptorState::Ready); + pin.Reset(); + EXPECT_TRUE(Manager->BeginTextureVersionWrite(texture)); + } + } + + void CheckTextureStagingAdmission(bool transfer) + { + const auto texture = CreateProducerTestTexture("bounded-texture-staging"); + ASSERT_TRUE(texture.Valid()); + RRMUploadBatchTestHelper::InitializeTextureUploadSlots(*Manager, *Device, UploadCommands, transfer); + TextureUploadSlotsInitialized = true; + auto* resource = Device->GlobalTextures.Access(texture); + ASSERT_NE(resource, nullptr); + auto* image = Device->ImageBufferManager.Access(resource->BufferHandle); + ASSERT_NE(image, nullptr); + const auto initial_layout = image->Layout; + const auto before = Device->GpuMem.GetAllocationStatistics(); + const auto original_limits = Device->GpuMem.GetAllocationLimits(); + auto limits = original_limits; + const auto domain = static_cast(GpuMemoryDomain::HostStaging); + limits.DomainBytes[domain] = before.DomainAdmittedBytes[domain]; + ASSERT_TRUE(Device->GpuMem.SetAllocationLimits(limits)); + uint8_t pixels[4] = {17, 53, 127, 255}; + EXPECT_FALSE(Manager->UploadTextureBuffer(0, 0, texture, pixels, sizeof(pixels)).Valid()); + EXPECT_EQ(image->Layout, initial_layout); + EXPECT_EQ(Commands[0]->GetState(), ZEngine::Hardwares::CommandBufferState::Idle); + EXPECT_EQ(RRMUploadBatchTestHelper::TextureUploadSlotValue(*Manager, transfer), 0u); + EXPECT_FALSE(RRMUploadBatchTestHelper::TextureUploadSlotHasStaging(*Manager, transfer)); + EXPECT_FALSE(RRMUploadBatchTestHelper::TextureUploadPending(*Manager, texture)); + EXPECT_EQ(Manager->GetTextureDecodeState(texture), RRM::TextureDecodeState::Untracked); + EXPECT_TRUE(Manager->CanWriteTextureVersion(texture)); + const auto refused = Device->GpuMem.GetAllocationStatistics(); + EXPECT_EQ(refused.AdmittedBytes, before.AdmittedBytes); + EXPECT_EQ(refused.ReservedRecords, before.ReservedRecords); + EXPECT_EQ(refused.AllocationCount, before.AllocationCount); + ASSERT_TRUE(Device->GpuMem.SetAllocationLimits(original_limits)); + ASSERT_TRUE(Manager->UploadTextureBuffer(0, 0, texture, pixels, sizeof(pixels)).Valid()); + EXPECT_TRUE(RRMUploadBatchTestHelper::TextureUploadPending(*Manager, texture)); + RRMUploadBatchTestHelper::ClearTextureUploadJobs(*Manager); + const auto reclaimed = Device->GpuMem.GetAllocationStatistics(); + EXPECT_EQ(reclaimed.AdmittedBytes, before.AdmittedBytes); + EXPECT_EQ(reclaimed.ReservedRecords, before.ReservedRecords); + EXPECT_EQ(reclaimed.AllocationCount, before.AllocationCount); + EXPECT_EQ(image->Layout, initial_layout); + } + void CheckRejectedTextureUploadReclamation(bool transfer) { const auto texture = CreateProducerTestTexture("rejected-texture-upload"); @@ -4368,10 +4717,9 @@ class RenderResourceManagerBatchTest : public GpuAllocatorTest EXPECT_EQ(after.total.statistics.allocationCount, before.total.statistics.allocationCount); EXPECT_EQ(after.total.statistics.allocationBytes, before.total.statistics.allocationBytes); } - // Queue-capacity rejection happens inline, before UploadTextureBuffer - // returns. It must not restore a layout belonging to discarded commands. + // Queue capacity is checked before claiming the image or recording. RRMUploadBatchTestHelper::SaturateTextureUploadQueue(*Manager); - ASSERT_TRUE(Manager->UploadTextureBuffer(0, 0, texture, pixels, sizeof(pixels)).Valid()); + EXPECT_FALSE(Manager->UploadTextureBuffer(0, 0, texture, pixels, sizeof(pixels)).Valid()); EXPECT_EQ(RRMUploadBatchTestHelper::TextureUploadSlotValue(*Manager, transfer), 0u); EXPECT_FALSE(RRMUploadBatchTestHelper::TextureUploadSlotHasStaging(*Manager, transfer)); const auto* resource = Device->GlobalTextures.Access(texture); @@ -4415,6 +4763,11 @@ class RenderResourceManagerBatchTest : public GpuAllocatorTest uint8_t pixels[4] = {17, 53, 127, 255}; // Invalid input must not leave a hold which would strand destruction. EXPECT_FALSE(Manager->UploadTextureBuffer(0, 0, texture, pixels, 5).Valid()); + for (size_t size = 1; size < sizeof(pixels); ++size) + { + EXPECT_FALSE(Manager->UploadTextureBuffer(0, 0, texture, pixels, size).Valid()); + EXPECT_FALSE(Device->WriteTextureData(Commands[0].get(), texture, pixels, nullptr, false, size)); + } EXPECT_FALSE(RRMUploadBatchTestHelper::TextureUploadPending(*Manager, texture)); EXPECT_TRUE(Manager->CanWriteTextureVersion(texture)); EXPECT_TRUE(RRMUploadBatchTestHelper::RefuseUploadWithBusyTextureSlots(*Manager, texture, pixels, sizeof(pixels), transfer)); @@ -4538,6 +4891,36 @@ class RenderResourceManagerBatchTest : public GpuAllocatorTest } }; +TEST_F(RenderResourceManagerBatchTest, CookedAssetImageUsesTextureBudgetDespiteAttachmentCapability) +{ + const auto regular = CreateProducerTestTexture("regular-target"); + ASSERT_TRUE(regular.Valid()); + const auto before = Device->GpuMem.GetAllocationStatistics(); + ZEngine::Rendering::Specifications::TextureSpecification specification = {}; + specification.Width = specification.Height = 1; + specification.Format = ZEngine::Rendering::Specifications::ImageFormat::R8G8B8A8_UNORM; + specification.IsCookedAsset = true; + const auto asset = Device->TryCreateTexture(specification, "cooked-asset"); + ASSERT_TRUE(asset.Valid()); + const auto after = Device->GpuMem.GetAllocationStatistics(); + const auto texture_domain = static_cast(GpuMemoryDomain::DeviceTexture); + const auto target_domain = static_cast(GpuMemoryDomain::RenderTarget); + EXPECT_GT(after.DomainAdmittedBytes[texture_domain], before.DomainAdmittedBytes[texture_domain]); + EXPECT_EQ(after.DomainAdmittedBytes[target_domain], before.DomainAdmittedBytes[target_domain]); + const auto original_limits = Device->GpuMem.GetAllocationLimits(); + auto limited = original_limits; + limited.DomainBytes[texture_domain] = after.DomainAdmittedBytes[texture_domain]; + ASSERT_TRUE(Device->GpuMem.SetAllocationLimits(limited)); + VkResult status = VK_SUCCESS; + const auto refused = Device->TryCreateTexture(specification, "cooked-asset-over-budget", 0, &status); + const bool restored = Device->GpuMem.SetAllocationLimits(original_limits); + EXPECT_TRUE(restored); + EXPECT_FALSE(refused.Valid()); + EXPECT_EQ(status, VK_ERROR_OUT_OF_DEVICE_MEMORY); + Device->DestroyTexture(asset); + Device->DestroyTexture(regular); +} + TEST_F(RenderResourceManagerBatchTest, BufferAliasExecutesAfterBackingRetiresAndKeepsChargeUntilCompletion) { const auto before = allocator().GetAllocationStatistics(); @@ -4661,6 +5044,39 @@ TEST_F(RenderResourceManagerBatchTest, InitialBufferBundleIsPreparedBeforeReadyA EXPECT_TRUE(Manager->ForgetResourceTimeline(second.get())); } +TEST_F(RenderResourceManagerBatchTest, DeviceLossDuringRetirementDoesNotImplyConsumerCompletion) +{ + RRM::SceneInitialFrameInput input = {}; + RRM::SceneResourcePreparationId id = {}; + ASSERT_TRUE(Manager->PrepareSceneResources({1, 1}, 1, {}, &id, RRM::ScenePreparationContent::MeshMaterials, nullptr, &input)); + ASSERT_EQ(FinishScenePreparation(id).State, RRM::SceneResourcePreparationState::ResourcesRetained); + ZEngine::Rendering::Scenes::PreparedSceneFrame frame = {}; + ASSERT_TRUE(Manager->GetFramePreparedSceneFrame(id, &frame)); + ASSERT_NE(frame.TransformBuffer, nullptr); + const auto allocations = LiveAllocationBytes(); + auto consumer = std::make_unique(Device.get(), true); + ASSERT_TRUE(Manager->ReleaseSceneResourcePreparation(id)); + Manager->NotifyResourceSubmission(0, consumer.get(), 4); + Manager->FinishResourceFrame(); + + // Simulate the host observing loss while an accepted consumer is pending. + // The real device remains healthy so the test can finish and clean up. + Device->IsDeviceLost.store(true, std::memory_order_release); + Manager->PollSceneResourcePreparations(); + RRM::SceneResourcePreparationResult result = {}; + EXPECT_TRUE(Manager->PollSceneResourcePreparation(id, &result)); + EXPECT_EQ(LiveAllocationBytes(), allocations); + EXPECT_FALSE(Manager->ForgetResourceTimeline(consumer.get())); + Device->IsDeviceLost.store(false, std::memory_order_release); + + consumer->Signal(4); + Manager->BeginFrame(0); + Manager->PollSceneResourcePreparations(); + EXPECT_FALSE(Manager->PollSceneResourcePreparation(id, &result)); + EXPECT_LT(LiveAllocationBytes(), allocations); + EXPECT_TRUE(Manager->ForgetResourceTimeline(consumer.get())); +} + TEST_F(RenderResourceManagerBatchTest, InitialBufferBundleSurvivesMailboxCancellationAndUnsubmittedRecordingReclaimsWithoutAGpuWait) { const auto baseline = LiveAllocationBytes(); @@ -6506,8 +6922,13 @@ TEST_F(RenderResourceManagerBatchTest, SceneTexturePreparationObservesProducerCo Swapchain.RenderTimelineNextValue = 1; Manager->EndFrame(); // Accepted producer work remains blocked on the host gate. RRMUploadBatchTestHelper::PublishSceneTextureProducer(*Manager, Memory.MainArena, {.Texture = texture, .CompletionTimeline = Timeline.get(), .CompletionValue = 1, .ProducerQueueFamily = Device->GraphicFamilyIndex}); + RRMUploadBatchTestHelper::MarkTextureSubmitted(*Manager, texture); // No fatal assertions may leave this accepted submission stranded. - bool failed = false; + bool failed = false; + RRM::TextureVersionPin prepared = {}; + uint64_t publication = 0; + EXPECT_EQ(Manager->PrepareSubmittedTextureVersion(texture, &prepared, &publication), ZEngine::Rendering::SceneTextureDescriptorState::Pending); + EXPECT_TRUE(prepared); EXPECT_FALSE(RRMUploadBatchTestHelper::SceneTextureReady(*Manager, retained, &failed)); EXPECT_FALSE(failed); // Accepted consumer work can wait on this still-blocked producer. Removing @@ -6515,6 +6936,8 @@ TEST_F(RenderResourceManagerBatchTest, SceneTexturePreparationObservesProducerCo const auto ticket = *Manager->FindStreamingUploadTicket(texture); Manager->AcknowledgeStreamingUploadTicket(ticket); EXPECT_EQ(Manager->FindStreamingUploadTicket(texture), nullptr); + EXPECT_EQ(Manager->PollSubmittedTextureAcquire(prepared), ZEngine::Rendering::SceneTextureDescriptorState::Failed); + EXPECT_EQ(Manager->PrepareSubmittedTextureVersion(texture, &prepared, &publication), ZEngine::Rendering::SceneTextureDescriptorState::Pending); EXPECT_FALSE(RRMUploadBatchTestHelper::SceneTextureReady(*Manager, retained, &failed)); EXPECT_FALSE(failed); gate->Signal(1); @@ -6524,6 +6947,7 @@ TEST_F(RenderResourceManagerBatchTest, SceneTexturePreparationObservesProducerCo EXPECT_EQ(Manager->FindStreamingUploadTicket(texture), nullptr); Swapchain.RenderTimeline = nullptr; Swapchain.RenderTimelineNextValue = 0; + prepared.Reset(); retained.Reset(); } @@ -6532,6 +6956,21 @@ TEST_F(RenderResourceManagerBatchTest, TextureReleaseRetainsAcknowledgedProducer CheckAcknowledgedProducerRetirement(false); } +TEST_F(RenderResourceManagerBatchTest, TextureConsumerAcquireNeedsCompletedGraphSubmission) +{ + CheckTextureConsumerAcquire(false); +} + +TEST_F(RenderResourceManagerBatchTest, CancelledTextureWaitsForAcceptedConsumerAcquire) +{ + CheckTextureConsumerAcquire(true); +} + +TEST_F(RenderResourceManagerBatchTest, TextureAcquireFromAnotherFamilyCannotQualifyGraphicsUse) +{ + CheckTextureConsumerAcquire(false, true); +} + TEST_F(RenderResourceManagerBatchTest, DirectTextureDestroyRetainsAcknowledgedProducerAcrossReleaseOwners) { CheckAcknowledgedProducerRetirement(true); @@ -6542,6 +6981,1411 @@ TEST_F(RenderResourceManagerBatchTest, RejectedGraphicsTextureJobsReclaimSlotsAn CheckRejectedTextureUploadReclamation(false); } +TEST_F(RenderResourceManagerBatchTest, GraphicsTextureStagingRefusalPreservesStateAndAllowsRetry) +{ + CheckTextureStagingAdmission(false); +} + +TEST_F(RenderResourceManagerBatchTest, RejectedTextureProducerCannotPrepareAnExactVersion) +{ + const auto texture = CreateProducerTestTexture("rejected-cooked-producer"); + ASSERT_TRUE(texture.Valid()); + RRMUploadBatchTestHelper::InitializeTextureUploadSlots(*Manager, *Device, UploadCommands, false); + TextureUploadSlotsInitialized = true; + uint8_t pixels[4] = {17, 53, 127, 255}; + ASSERT_TRUE(Manager->UploadTextureBuffer(0, 0, texture, pixels, sizeof(pixels)).Valid()); + RRM::TextureVersionPin pin = {}; + uint64_t publication = 0; + EXPECT_EQ(Manager->PrepareSubmittedTextureVersion(texture, &pin, &publication), ZEngine::Rendering::SceneTextureDescriptorState::Pending); + EXPECT_FALSE(pin); + RRMUploadBatchTestHelper::ClearTextureUploadJobs(*Manager); + EXPECT_EQ(Manager->PrepareSubmittedTextureVersion(texture, &pin, &publication), ZEngine::Rendering::SceneTextureDescriptorState::Failed); + EXPECT_FALSE(pin); + EXPECT_EQ(publication, 0u); +} + +TEST_F(RenderResourceManagerBatchTest, TextureStagingSlicesAndCancellationKeepCreditsBounded) +{ + using Staging = RRM::UploadStaging; + const auto before = Device->GpuMem.GetAllocationStatistics(); + Staging staging = {}; + std::vector pixels(Staging::MaxSliceBytes + 17, 0x5a); + ASSERT_TRUE(Manager->PrepareUploadStaging(pixels.size(), &staging)); + EXPECT_FALSE(staging.Ready()); + size_t consumed = 123; + EXPECT_FALSE(staging.Append({}, &consumed)); + EXPECT_EQ(consumed, 0u); + bool worker_success = false; + std::thread worker([&] { worker_success = staging.Append(pixels, &consumed); }); + worker.join(); + ASSERT_TRUE(worker_success); + EXPECT_EQ(consumed, Staging::MaxSliceBytes); + EXPECT_EQ(staging.Written(), Staging::MaxSliceBytes); + EXPECT_FALSE(staging.Ready()); + EXPECT_FALSE(Manager->PrepareUploadStaging(0, &staging)); + EXPECT_EQ(staging.Written(), Staging::MaxSliceBytes); + Staging moved = std::move(staging); + EXPECT_EQ(staging.Size(), 0u); + ASSERT_TRUE(moved.Append({pixels.data() + consumed, pixels.size() - consumed}, &consumed)); + EXPECT_EQ(consumed, 17u); + EXPECT_TRUE(moved.Ready()); + EXPECT_FALSE(moved.Append(pixels, &consumed)); + EXPECT_EQ(consumed, 0u); + moved.Reset(); + moved.Reset(); + const auto after = Device->GpuMem.GetAllocationStatistics(); + EXPECT_EQ(after.AdmittedBytes, before.AdmittedBytes); + EXPECT_EQ(after.ReservedRecords, before.ReservedRecords); + EXPECT_EQ(after.AllocationCount, before.AllocationCount); +} + +TEST_F(RenderResourceManagerBatchTest, CookedMeshStagingSlicesBothStreamsAndReclaimsOnCancellation) +{ + using Staging = ZEngine::Rendering::Meshes::CookedMeshStaging; + ZEngine::Importers::AssetMesh mesh = {}; + constexpr size_t vertex_floats = RRM::UploadStaging::MaxSliceBytes / sizeof(float) + 8; + mesh.Vertices.init(&Memory.MainArena, vertex_floats, vertex_floats); + for (size_t index = 0; index < vertex_floats; ++index) + { + mesh.Vertices[index] = static_cast(index % 97); + } + mesh.Indices.init(&Memory.MainArena, 3); + mesh.Indices.push(2); + mesh.Indices.push(1); + mesh.Indices.push(0); + Staging staging = {}; + RRMUploadBatchTestHelper::SetMeshSliceTestSource(staging, mesh); + const auto before = Device->GpuMem.GetAllocationStatistics(); + const auto original_limits = Device->GpuMem.GetAllocationLimits(); + auto limits = original_limits; + const auto domain = static_cast(GpuMemoryDomain::HostStaging); + limits.DomainBytes[domain] = before.DomainAdmittedBytes[domain]; + ASSERT_TRUE(Device->GpuMem.SetAllocationLimits(limits)); + VkResult status = VK_SUCCESS; + EXPECT_FALSE(staging.Prepare(*Manager, &status)); + EXPECT_EQ(status, VK_ERROR_OUT_OF_DEVICE_MEMORY); + EXPECT_EQ(staging.State(), Staging::Phase::SourceReady); + ASSERT_TRUE(Device->GpuMem.SetAllocationLimits(original_limits)); + ASSERT_TRUE(staging.Prepare(*Manager)); + bool staged = false; + std::thread worker([&] { staged = staging.StageOne(); }); + worker.join(); + ASSERT_TRUE(staged); + EXPECT_EQ(staging.StagedBytes(), RRM::UploadStaging::MaxSliceBytes); + EXPECT_EQ(staging.State(), Staging::Phase::Staging); + ASSERT_TRUE(staging.StageOne()); + EXPECT_EQ(staging.StagedBytes(), vertex_floats * sizeof(float)); + EXPECT_EQ(staging.State(), Staging::Phase::Staging); + ASSERT_TRUE(staging.StageOne()); + EXPECT_EQ(staging.State(), Staging::Phase::Staged); + EXPECT_EQ(staging.StagedBytes(), staging.VertexBytes() + staging.IndexBytes()); + EXPECT_FALSE(staging.StageOne()); + const auto& buffer = RRMUploadBatchTestHelper::MeshStagingBuffer(staging); + std::vector bytes(staging.VertexBytes() + staging.IndexBytes()); + ASSERT_EQ(vmaCopyAllocationToMemory(Device->GpuMem.Allocator, buffer.Allocation, 0, bytes.data(), bytes.size()), VK_SUCCESS); + const auto* vertices = reinterpret_cast(mesh.Vertices.data()); + const auto* indices = reinterpret_cast(mesh.Indices.data()); + EXPECT_TRUE(std::equal(vertices, vertices + staging.VertexBytes(), bytes.data())); + EXPECT_TRUE(std::equal(indices, indices + staging.IndexBytes(), bytes.data() + staging.VertexBytes())); + staging.Reset(); + staging.Reset(); + EXPECT_EQ(staging.State(), Staging::Phase::Empty); + const auto after = Device->GpuMem.GetAllocationStatistics(); + EXPECT_EQ(after.AdmittedBytes, before.AdmittedBytes); + EXPECT_EQ(after.ReservedRecords, before.ReservedRecords); + EXPECT_EQ(after.AllocationCount, before.AllocationCount); +} + +class CookedMeshUploadTest : public RenderResourceManagerBatchTest +{ +protected: + using Staging = ZEngine::Rendering::Meshes::CookedMeshStaging; + using Admission = RRM::CookedMeshAdmission; + using State = RRM::CookedMeshUploadState; + TLSFSlab Metadata = {}; + BufferView Vertices = {}; + BufferView Indices = {}; + ZEngine::Importers::AssetMesh Mesh = {}; + std::unique_ptr Gate = {}; + bool GateSignalled = false; + bool GeometryInitialized = false; + + void SetUp() override + { + RenderResourceManagerBatchTest::SetUp(); + if (!Initialized) + { + return; + } + Metadata.Init(&Memory.MainArena, 16 * 1024); + Vertices = allocator().AllocateBuffer(1024, VK_BUFFER_USAGE_TRANSFER_DST_BIT, GpuMemoryDomain::HostReadback, "cooked-vertices"); + Indices = allocator().AllocateBuffer(1024, VK_BUFFER_USAGE_TRANSFER_DST_BIT, GpuMemoryDomain::HostReadback, "cooked-indices"); + RRMUploadBatchTestHelper::InitializeCookedGeometry(*Manager, Memory.MainArena, Vertices, Indices); + GeometryInitialized = true; + Mesh.Vertices.init(&Memory.MainArena, 8, 8); + for (uint32_t i = 0; i < 8; ++i) + { + Mesh.Vertices[i] = float(i + 1); + } + Mesh.Indices.init(&Memory.MainArena, 1); + Mesh.Indices.push(17); + Mesh.BoundsRadius = 3.f; + } + + void TearDown() override + { + if (GeometryInitialized) + { + if (Gate && !GateSignalled) + { + Gate->Signal(1); + } + Manager->EndFrame(); + vkDeviceWaitIdle(s_vk->Device); + RRMUploadBatchTestHelper::ReleaseCookedGeometryAfterIdle(*Manager); + Swapchain.RenderTimeline = nullptr; + Swapchain.RenderTimelineNextValue = 0; + Gate.reset(); + allocator().FreeBuffer(Vertices); + allocator().FreeBuffer(Indices); + Metadata.Shutdown(); + } + RenderResourceManagerBatchTest::TearDown(); + } + + bool Stage(Staging& staging) + { + RRMUploadBatchTestHelper::SetMeshSliceTestSource(staging, Mesh); + return staging.Prepare(*Manager) && staging.StageOne() && staging.StageOne(); + } + + void DelaySubmission() + { + Gate = std::make_unique(Device.get(), true); + Swapchain.RenderTimeline = Gate.get(); + Swapchain.RenderTimelineNextValue = 1; + } + + void Unblock() + { + Gate->Signal(1); + GateSignalled = true; + } +}; + +TEST_F(CookedMeshUploadTest, CopiesBytesAndRetainsExactMetadataUntilTaken) +{ + Staging staging; + ASSERT_TRUE(Stage(staging)); + RRM::CookedMeshUploadId id = {}; + ASSERT_EQ(Manager->RecordCookedMeshUpload(0, staging, Metadata, &id), Admission::Recorded); + EXPECT_EQ(Manager->PollCookedMeshUpload(id).State, State::Pending); + DelaySubmission(); + Manager->EndFrame(); + EXPECT_EQ(Manager->PollCookedMeshUpload(id).State, State::Pending); + Unblock(); + Timeline->Wait(1); + const auto completed = Manager->PollCookedMeshUpload(id); + ASSERT_EQ(completed.State, State::Ready); + RRM::CookedMeshVersion version = {}; + ASSERT_TRUE(Manager->TakeCookedMeshUpload(id, &version)); + EXPECT_EQ(version.Identity(), completed.Identity); + EXPECT_EQ(version.Digest(), completed.Digest); + EXPECT_TRUE(version.Identity().IsValid()); + EXPECT_EQ(version.Digest().Bytes[0], 42); + EXPECT_FALSE(Manager->TakeCookedMeshUpload(id, &version)); + RRM::CookedMeshVersion moved = std::move(version); + EXPECT_FALSE(version); + ASSERT_TRUE(moved.Geometry()); + EXPECT_FLOAT_EQ(moved.Geometry()->DrawMetadata->BoundsRadius, 3.f); + std::array vertices = {}; + uint32_t index = 0; + ASSERT_EQ(vmaCopyAllocationToMemory(allocator().Allocator, Vertices.Allocation, moved.Geometry()->Region.VtxByteOffset, vertices.data(), sizeof(vertices)), VK_SUCCESS); + ASSERT_EQ(vmaCopyAllocationToMemory(allocator().Allocator, Indices.Allocation, moved.Geometry()->Region.IdxByteOffset, &index, sizeof(index)), VK_SUCCESS); + EXPECT_TRUE(std::equal(vertices.begin(), vertices.end(), Mesh.Vertices.data())); + EXPECT_EQ(index, 17u); + EXPECT_EQ(RRMUploadBatchTestHelper::GeometryBytes(*Manager), 32u); + moved.Reset(); + EXPECT_EQ(RRMUploadBatchTestHelper::GeometryBytes(*Manager), 0u); +} + +TEST_F(CookedMeshUploadTest, CancellationKeepsAcceptedCopiesPinnedUntilCompletion) +{ + Staging staging; + ASSERT_TRUE(Stage(staging)); + RRM::CookedMeshUploadId id = {}; + ASSERT_EQ(Manager->RecordCookedMeshUpload(0, staging, Metadata, &id), Admission::Recorded); + ASSERT_TRUE(Manager->CancelCookedMeshUpload(id)); + EXPECT_TRUE(Manager->PollCookedMeshUpload(id).CleanupPending); + DelaySubmission(); + Manager->EndFrame(); + EXPECT_FALSE(Manager->ReleaseCookedMeshUpload(id)); + EXPECT_EQ(RRMUploadBatchTestHelper::GeometryBytes(*Manager), 32u); + Unblock(); + Timeline->Wait(1); + const auto result = Manager->PollCookedMeshUpload(id); + EXPECT_EQ(result.State, State::Cancelled); + EXPECT_FALSE(result.CleanupPending); + EXPECT_EQ(RRMUploadBatchTestHelper::GeometryBytes(*Manager), 0u); + EXPECT_TRUE(Manager->ReleaseCookedMeshUpload(id)); +} + +TEST_F(CookedMeshUploadTest, CancellationAfterSubmissionWaitsForTheGpu) +{ + Staging staging; + ASSERT_TRUE(Stage(staging)); + RRM::CookedMeshUploadId id = {}; + ASSERT_EQ(Manager->RecordCookedMeshUpload(0, staging, Metadata, &id), Admission::Recorded); + DelaySubmission(); + Manager->EndFrame(); + ASSERT_TRUE(Manager->CancelCookedMeshUpload(id)); + const auto pending = Manager->PollCookedMeshUpload(id); + EXPECT_EQ(pending.State, State::Cancelled); + EXPECT_TRUE(pending.CleanupPending); + EXPECT_EQ(RRMUploadBatchTestHelper::GeometryBytes(*Manager), 32u); + EXPECT_FALSE(Manager->ReleaseCookedMeshUpload(id)); + Unblock(); + Timeline->Wait(1); + EXPECT_FALSE(Manager->PollCookedMeshUpload(id).CleanupPending); + EXPECT_EQ(RRMUploadBatchTestHelper::GeometryBytes(*Manager), 0u); + EXPECT_TRUE(Manager->ReleaseCookedMeshUpload(id)); +} + +TEST_F(CookedMeshUploadTest, ResetInvalidatesReadyResultWithoutReleasingReplacement) +{ + Staging staging; + ASSERT_TRUE(Stage(staging)); + RRM::CookedMeshUploadId id = {}; + ASSERT_EQ(Manager->RecordCookedMeshUpload(0, staging, Metadata, &id), Admission::Recorded); + Manager->EndFrame(); + Timeline->Wait(1); + ASSERT_EQ(Manager->PollCookedMeshUpload(id).State, State::Ready); + RRMUploadBatchTestHelper::ResetGeometry(*Manager); + const auto replacement = RRMUploadBatchTestHelper::InstallGeometry(*Manager); + ASSERT_TRUE(replacement.IsValid()); + ZEngine::Rendering::BufferHandle handle = {}; + RRM::MeshGeometryPin pin = {}; + EXPECT_FALSE(Manager->TakeCookedMeshUpload(id, &handle, &pin)); + EXPECT_EQ(Manager->PollCookedMeshUpload(id).State, State::Failed); + EXPECT_TRUE(Manager->ReleaseCookedMeshUpload(id)); + EXPECT_EQ(RRMUploadBatchTestHelper::GeometryBytes(*Manager), 32u); + Manager->Release(replacement); +} + +TEST_F(CookedMeshUploadTest, FullRecordTableRefusesWithoutConsumingStaging) +{ + std::array ids = {}; + for (auto& id : ids) + { + Staging staging; + ASSERT_TRUE(Stage(staging)); + ASSERT_EQ(Manager->RecordCookedMeshUpload(0, staging, Metadata, &id), Admission::Recorded); + } + Staging extra; + ASSERT_TRUE(Stage(extra)); + RRM::CookedMeshUploadId extra_id = {}; + EXPECT_EQ(Manager->RecordCookedMeshUpload(0, extra, Metadata, &extra_id), Admission::Retry); + EXPECT_EQ(extra.State(), Staging::Phase::Staged); + for (const auto& id : ids) + { + EXPECT_TRUE(Manager->CancelCookedMeshUpload(id)); + } + Manager->EndFrame(); + Timeline->Wait(1); + for (const auto& id : ids) + { + EXPECT_TRUE(Manager->ReleaseCookedMeshUpload(id)); + } + EXPECT_EQ(RRMUploadBatchTestHelper::GeometryBytes(*Manager), 0u); +} + +TEST_F(CookedMeshUploadTest, BusyBatchSlotReturnsRetryWithoutConsumingStaging) +{ + Staging first; + ASSERT_TRUE(Stage(first)); + RRM::CookedMeshUploadId id = {}; + ASSERT_EQ(Manager->RecordCookedMeshUpload(0, first, Metadata, &id), Admission::Recorded); + DelaySubmission(); + Manager->EndFrame(); + Staging second; + ASSERT_TRUE(Stage(second)); + RRM::CookedMeshUploadId second_id = {}; + EXPECT_EQ(Manager->RecordCookedMeshUpload(0, second, Metadata, &second_id), Admission::Retry); + EXPECT_EQ(second.State(), Staging::Phase::Staged); + Unblock(); + Timeline->Wait(1); + Manager->BeginFrame(0); + ASSERT_EQ(Manager->RecordCookedMeshUpload(0, second, Metadata, &second_id), Admission::Recorded); + EXPECT_TRUE(Manager->CancelCookedMeshUpload(id)); + EXPECT_TRUE(Manager->CancelCookedMeshUpload(second_id)); + Manager->EndFrame(); + Timeline->Wait(2); + EXPECT_TRUE(Manager->ReleaseCookedMeshUpload(id)); + EXPECT_TRUE(Manager->ReleaseCookedMeshUpload(second_id)); +} + +TEST_F(CookedMeshUploadTest, MetadataAndPoolRefusalPreserveStaging) +{ + Staging staging; + ASSERT_TRUE(Stage(staging)); + TLSFSlab unavailable = {}; + RRM::CookedMeshUploadId id = {}; + EXPECT_EQ(Manager->RecordCookedMeshUpload(0, staging, unavailable, &id), Admission::Failed); + EXPECT_EQ(staging.State(), Staging::Phase::Staged); + std::array ids = {}; + for (auto& recorded : ids) + { + Staging candidate; + ASSERT_TRUE(Stage(candidate)); + ASSERT_EQ(Manager->RecordCookedMeshUpload(0, candidate, Metadata, &recorded), Admission::Recorded); + } + // The bounded table is full; retire its records while leaving the geometry + // registered through transferred pins, then exhaust the geometry pool. + Manager->EndFrame(); + Timeline->Wait(1); + std::array handles = {}; + std::array pins = {}; + for (size_t i = 0; i < ids.size(); ++i) + { + ASSERT_TRUE(Manager->TakeCookedMeshUpload(ids[i], &handles[i], &pins[i])); + } + // This pool has 1024 vertex bytes; another sixteen 32-byte versions fill it. + Manager->BeginFrame(1); + for (auto& recorded : ids) + { + Staging candidate; + ASSERT_TRUE(Stage(candidate)); + ASSERT_EQ(Manager->RecordCookedMeshUpload(1, candidate, Metadata, &recorded), Admission::Recorded); + } + Manager->EndFrame(); + Timeline->Wait(2); + for (const auto& recorded : ids) + { + ASSERT_EQ(Manager->PollCookedMeshUpload(recorded).State, State::Ready); + } + // Free a table slot without reclaiming its region. + ZEngine::Rendering::BufferHandle last_handle = {}; + RRM::MeshGeometryPin last_pin = {}; + ASSERT_TRUE(Manager->TakeCookedMeshUpload(ids[0], &last_handle, &last_pin)); + EXPECT_EQ(Manager->RecordCookedMeshUpload(2, staging, Metadata, &id), Admission::Failed); + EXPECT_EQ(staging.State(), Staging::Phase::Staged); + Manager->Release(last_handle); + last_pin.Reset(); + for (size_t i = 0; i < handles.size(); ++i) + { + Manager->Release(handles[i]); + pins[i].Reset(); + } + for (size_t i = 1; i < ids.size(); ++i) + { + EXPECT_TRUE(Manager->ReleaseCookedMeshUpload(ids[i])); + } +} + +TEST_F(RenderResourceManagerBatchTest, GraphicsTextureStagingUploadsTheSuppliedPixels) +{ + const auto texture = CreateProducerTestTexture("texture-staging-readback", true); + ASSERT_TRUE(texture.Valid()); + RRMUploadBatchTestHelper::InitializeTextureUploadSlots(*Manager, *Device, UploadCommands, false); + TextureUploadSlotsInitialized = true; + uint8_t pixels[4] = {17, 53, 127, 255}; + RRM::UploadStaging wrong_size = {}; + ASSERT_TRUE(Manager->PrepareUploadStaging(8, &wrong_size)); + const uint8_t longer[8] = {}; + size_t consumed = 0; + ASSERT_TRUE(wrong_size.Append(longer, &consumed)); + EXPECT_FALSE(Manager->UploadPreparedTextureBuffer(0, 0, texture, wrong_size).Valid()); + EXPECT_TRUE(wrong_size.Ready()); + EXPECT_FALSE(RRMUploadBatchTestHelper::TextureUploadPending(*Manager, texture)); + wrong_size.Reset(); + RRM::UploadStaging staging = {}; + ASSERT_TRUE(Manager->PrepareUploadStaging(sizeof(pixels), &staging)); + ASSERT_TRUE(staging.Append({pixels, 1}, &consumed)); + EXPECT_EQ(consumed, 1u); + EXPECT_FALSE(Manager->UploadPreparedTextureBuffer(0, 0, texture, staging).Valid()); + EXPECT_EQ(staging.Written(), 1u); + EXPECT_FALSE(RRMUploadBatchTestHelper::TextureUploadPending(*Manager, texture)); + ASSERT_TRUE(staging.Append({pixels + 1, 3}, &consumed)); + EXPECT_TRUE(staging.Ready()); + RRMUploadBatchTestHelper::SaturateTextureUploadQueue(*Manager); + EXPECT_FALSE(Manager->UploadPreparedTextureBuffer(0, 0, texture, staging).Valid()); + EXPECT_TRUE(staging.Ready()); + EXPECT_FALSE(RRMUploadBatchTestHelper::TextureUploadPending(*Manager, texture)); + RRMUploadBatchTestHelper::ClearTextureUploadJobs(*Manager); + ASSERT_TRUE(Manager->UploadPreparedTextureBuffer(0, 0, texture, staging).Valid()); + EXPECT_FALSE(staging.Ready()); + EXPECT_EQ(staging.Size(), 0u); + Manager->SubmitAsyncUploads(); + const auto* ticket = Manager->FindStreamingUploadTicket(texture); + ASSERT_NE(ticket, nullptr); + ticket->CompletionTimeline->Wait(ticket->CompletionValue); + RRMUploadBatchTestHelper::RetireTextureUploadSlots(*Manager); + + CheckTextureReadback(texture, pixels); +} + +TEST_F(RenderResourceManagerBatchTest, CookedTextureStagesVerifiedPixelsAndSubmitsItsExactVersion) +{ + namespace Cook = ZEngine::Cook; + namespace VFS = ZEngine::Core::VFS; + namespace ECS = ZEngine::ECS; + using Cache = ZEngine::Managers::CookedCpuClosureCache; + using Upload = ZEngine::Rendering::Textures::CookedTextureUpload; + using JSON = nlohmann::json; + MemoryManager cook_memory = {}; + cook_memory.Initialize(ZMega(8), {}); + auto allocator = std::make_unique>(); + ECS::SceneComponentSchemaRegistry schemas = {}; + ECS::SceneDiagnostics scene_diagnostics = {}; + schemas.Initialize(&cook_memory.MainArena); + ASSERT_TRUE(ECS::RegisterBuiltInSceneComponentSchemas(schemas, &scene_diagnostics)); + ASSERT_TRUE(schemas.FreezeForWorker(&scene_diagnostics)); + VFS::VFSMemoryBackend backend = {}; + ASSERT_TRUE(backend.InitializeOwned(ZMega(2), 128).Succeeded()); + ASSERT_TRUE(backend.CreateDir(VFS::VFSPath::Parse("/output").Value()).Succeeded()); + ASSERT_TRUE(backend.CreateDir(VFS::VFSPath::Parse("/output/objects").Value()).Succeeded()); + ASSERT_TRUE(backend.CreateDir(VFS::VFSPath::Parse("/assets").Value()).Succeeded()); + VFS::VFSContext context = {}; + context.Initialize(&cook_memory.MainArena); + ASSERT_TRUE(context.Mount(&backend, VFS::VFSPath::Root(), 0).Succeeded()); + struct ContextGuard + { + VFS::VFSContext& Context; + ~ContextGuard() + { + Context.Shutdown(); + } + } context_guard{context}; + + JSON manifest = JSON::parse(Cook::Tests::ManifestGolden.begin(), Cook::Tests::ManifestGolden.end()); + const auto words = Cook::CookSchemaKeyWorkspaceWords(schemas); + ASSERT_TRUE(words.Succeeded()); + std::vector workspace(words.Value()); + Cook::Digest256 schema_key = {}; + ASSERT_TRUE(Cook::ComputeCookSchemaKey(*allocator, schemas, {workspace.data(), workspace.size()}, &schema_key).Succeeded()); + const auto schema_hex = schema_key.Hex(); + manifest["schema_key"] = std::string(schema_hex.data(), 64); + const std::array bmp = {0x42, 0x4d, 58, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 40, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 24, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 20, 30, 0}; + ZEngine::Core::Hash::SHA256 hash = {}; + Cook::Digest256 digest = {}; + ASSERT_TRUE(hash.Update({bmp.data(), bmp.size()}).Succeeded()); + ASSERT_TRUE(hash.Finalize(&digest).Succeeded()); + Cook::CookObjectDescriptor object = {}; + ASSERT_TRUE(Cook::MakeCookObjectDescriptor(digest, bmp.size(), "bmp", &object).Succeeded()); + const std::string object_path = std::string("/output/") + object.Path.CStr(); + ASSERT_TRUE(backend.WriteFile(VFS::VFSPath::Parse(object_path.c_str()).Value(), {bmp.data(), bmp.size()}).Succeeded()); + const auto texture_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000002"); + const auto scene_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000001"); + const auto material_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000003"); + const auto mesh_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000004"); + const auto environment_id = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000006"); + const Cook::CookAssetIdentity identity = {Cook::CookAssetKind::Texture, texture_id}; + const Cook::CookAssetIdentity material_identity = {Cook::CookAssetKind::Material, material_id}; + const Cook::CookAssetIdentity environment_identity = {Cook::CookAssetKind::EnvironmentMap, environment_id}; + ZEngine::Importers::AssetCodec::EnvironmentMapFileHeader environment_header = {.MagicNumber = ZENVMAP_MAGIC, .Version = 2, .HeaderByteSize = sizeof(environment_header), .ImporterVersion = 1, .SourceHash = 1, .FaceWidth = 1, .FaceHeight = 1, .Channel = 4, .LayerCount = 6, .MipCount = 1, .ColorSpace = 1, .Orientation = 1, .MipPolicy = 1, .Exposure = 1, .BufferByteSize = 96}; + std::array environment_bytes = {}; + std::memcpy(environment_bytes.data(), &environment_header, sizeof(environment_header)); + for (uint32_t i = 0; i < 24; ++i) + { + const float value = float(i + 1); + std::memcpy(environment_bytes.data() + sizeof(environment_header) + i * sizeof(float), &value, sizeof(value)); + } + ZEngine::Core::Hash::SHA256 environment_hash = {}; + Cook::Digest256 environment_digest = {}; + ASSERT_TRUE(environment_hash.Update({environment_bytes.data(), environment_bytes.size()}).Succeeded()); + ASSERT_TRUE(environment_hash.Finalize(&environment_digest).Succeeded()); + Cook::CookObjectDescriptor environment_object = {}; + ASSERT_TRUE(Cook::MakeCookObjectDescriptor(environment_digest, environment_bytes.size(), "zenvmap", &environment_object).Succeeded()); + const std::string environment_path = std::string("/output/") + environment_object.Path.CStr(); + ASSERT_TRUE(backend.WriteFile(VFS::VFSPath::Parse(environment_path.c_str()).Value(), {environment_bytes.data(), environment_bytes.size()}).Succeeded()); + ZEngine::Importers::AssetMesh mesh = {}; + mesh.MeshUUID = mesh_id; + mesh.Vertices.init(&cook_memory.MainArena, 24, 24); + mesh.Vertices[8] = 1.f; + mesh.Vertices[17] = 1.f; + mesh.Indices.init(&cook_memory.MainArena, 3); + mesh.Indices.push(0); + mesh.Indices.push(1); + mesh.Indices.push(2); + mesh.SubMeshes.init(&cook_memory.MainArena, 1); + mesh.SubMeshes.push({.MaterialUUID = material_id, .VertexCount = 3, .IndexCount = 3, .VertexUnitStreamSize = 32, .IndexUnitStreamSize = 4, .TotalByteSize = 108}); + ZEngine::Importers::AssetNodeHierarchy hierarchy = {}; + hierarchy.NodeHierarchyUUID = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000005"); + hierarchy.MeshUUID = mesh_id; + hierarchy.Hierarchies.init(&cook_memory.MainArena, 1); + hierarchy.LocalTransforms.init(&cook_memory.MainArena, 1); + hierarchy.GlobalTransforms.init(&cook_memory.MainArena, 1); + hierarchy.Names.init(&cook_memory.MainArena, 1); + hierarchy.MaterialNames.init(&cook_memory.MainArena, 0); + hierarchy.NodeNames.init(&cook_memory.MainArena, 16); + hierarchy.NodeMeshes.init(&cook_memory.MainArena, 16); + hierarchy.NodeMaterials.init(&cook_memory.MainArena, 16); + ASSERT_EQ(ZEngine::Importers::AddNode(hierarchy, -1, 0), 0); + hierarchy.Names.push_use({}).init(&cook_memory.MainArena, "Root"); + hierarchy.NodeNames.insert(0, 0); + hierarchy.NodeMeshes.insert(0, 0); + ZEngine::Importers::AssetCodec::ImportConfiguration config = {}; + config.VFS = &context; + config.OutputAssetsPath.init(&cook_memory.MainArena, "/assets"); + config.OutputAssetFile.init(&cook_memory.MainArena, "mesh.zemesh"); + ASSERT_TRUE(ZEngine::Importers::AssetCodec::SerializeMeshAssetFile(&cook_memory.MainArena, mesh, hierarchy, config).Succeeded()); + auto mesh_file = context.Open(VFS::VFSPath::Parse("/assets/mesh.zemesh").Value(), VFS::VFSOpenFlags::Read); + ASSERT_TRUE(mesh_file.Succeeded()); + auto mesh_size = mesh_file.Value()->Size(); + ASSERT_TRUE(mesh_size.Succeeded()); + std::vector mesh_bytes(static_cast(mesh_size.Value())); + ASSERT_TRUE(mesh_file.Value()->ReadAll({mesh_bytes.data(), mesh_bytes.size()}).Succeeded()); + ASSERT_TRUE(context.CloseChecked(mesh_file.Value()).Succeeded()); + ZEngine::Core::Hash::SHA256 mesh_hash = {}; + Cook::Digest256 mesh_digest = {}; + ASSERT_TRUE(mesh_hash.Update({mesh_bytes.data(), mesh_bytes.size()}).Succeeded()); + ASSERT_TRUE(mesh_hash.Finalize(&mesh_digest).Succeeded()); + Cook::CookObjectDescriptor mesh_object = {}; + ASSERT_TRUE(Cook::MakeCookObjectDescriptor(mesh_digest, mesh_bytes.size(), "zemesh", &mesh_object).Succeeded()); + const std::string mesh_path = std::string("/output/") + mesh_object.Path.CStr(); + ASSERT_TRUE(backend.WriteFile(VFS::VFSPath::Parse(mesh_path.c_str()).Value(), {mesh_bytes.data(), mesh_bytes.size()}).Succeeded()); + std::string material_bytes = + R"({"material_version":1,"uuid":"00000000-0000-4000-8000-000000000003","name":"Cooked","textures":{"albedo":{"uuid":"00000000-0000-4000-8000-000000000002","path":"assets/pixel.bmp"},"emissive":{"uuid":"00000000-0000-0000-0000-000000000000","path":""},"normal":{"uuid":"00000000-0000-0000-0000-000000000000","path":""},"opacity":{"uuid":"00000000-0000-0000-0000-000000000000","path":""},"specular":{"uuid":"00000000-0000-0000-0000-000000000000","path":""}},"ambient_color":[0,0,0,0],"albedo_color":[1,0.5,1,1],"emissive_color":[0,0,0,1],"roughness_color":[1,0.5,0,0],"specular_color":[0,0,0,0],"factors":[0,0,0.5,0],"alpha_mode":"BLEND","alpha_cutoff":0.5})"; + material_bytes.replace(material_bytes.find("assets/pixel.bmp"), sizeof("assets/pixel.bmp") - 1, object.Path.CStr()); + ZEngine::Core::Hash::SHA256 material_hash = {}; + Cook::Digest256 material_digest = {}; + ASSERT_TRUE(material_hash.Update({reinterpret_cast(material_bytes.data()), material_bytes.size()}).Succeeded()); + ASSERT_TRUE(material_hash.Finalize(&material_digest).Succeeded()); + Cook::CookObjectDescriptor material_object = {}; + ASSERT_TRUE(Cook::MakeCookObjectDescriptor(material_digest, material_bytes.size(), "zematerial", &material_object).Succeeded()); + const std::string material_path = std::string("/output/") + material_object.Path.CStr(); + ASSERT_TRUE(backend.WriteFile(VFS::VFSPath::Parse(material_path.c_str()).Value(), {reinterpret_cast(material_bytes.data()), material_bytes.size()}).Succeeded()); + const auto object_hex = digest.Hex(); + const auto material_hex = material_digest.Hex(); + manifest["assets"] = JSON::array({ + {{"kind", "material"}, {"uuid", uuids::to_string(material_id)}, {"format", "zematerial-v1-json"}, {"adapter_revision", 1}, {"input_key", std::string(64, '2')}, {"object", {{"path", material_object.Path.CStr()}, {"digest", std::string(material_hex.data(), 64)}, {"size", material_object.Size}}}, {"dependencies", JSON::array({{{"kind", "texture"}, {"uuid", uuids::to_string(texture_id)}, {"slot", "albedo"}}})}}, + { {"kind", "texture"}, {"uuid", uuids::to_string(texture_id)}, {"format", "image-bmp"}, {"adapter_revision", 1}, {"input_key", std::string(64, '1')}, {"object", {{"path", object.Path.CStr()}, {"digest", std::string(object_hex.data(), 64)}, {"size", object.Size}}}, {"dependencies", JSON::array()}} + }); + const auto mesh_hex = mesh_digest.Hex(); + manifest["assets"].push_back({ + {"kind", "mesh"}, + {"uuid", uuids::to_string(mesh_id)}, + {"format", "zemesh-v1-le64"}, + {"adapter_revision", 1}, + {"input_key", std::string(64, '3')}, + {"object", {{"path", mesh_object.Path.CStr()}, {"digest", std::string(mesh_hex.data(), 64)}, {"size", mesh_object.Size}}}, + {"dependencies", JSON::array({{{"kind", "material"}, {"uuid", uuids::to_string(material_id)}, {"slot", "submeshes/0/material"}}})} + }); + const auto environment_hex = environment_digest.Hex(); + manifest["assets"].push_back({ + {"kind", "environment-map"}, + {"uuid", uuids::to_string(environment_id)}, + {"format", "zenvmap-v2-le64"}, + {"adapter_revision", 1}, + {"input_key", std::string(64, '4')}, + {"object", {{"path", environment_object.Path.CStr()}, {"digest", std::string(environment_hex.data(), 64)}, {"size", environment_object.Size}}}, + {"dependencies", JSON::array()} + }); + std::sort(manifest["assets"].begin(), manifest["assets"].end(), [](const JSON& a, const JSON& b) { return a["kind"].get() < b["kind"].get(); }); + manifest["scenes"][0]["roots"] = JSON::array({ + {{"kind", "environment-map"}, {"uuid", uuids::to_string(environment_id)}}, + { {"kind", "mesh"}, {"uuid", uuids::to_string(mesh_id)}} + }); + const auto text = manifest.dump(); + ASSERT_TRUE(backend.WriteFile(VFS::VFSPath::Parse("/output/manifest.json").Value(), {reinterpret_cast(text.data()), text.size()}).Succeeded()); + VFS::VFSReadRootLease root = {}; + ASSERT_TRUE(context.AcquireReadRoot(VFS::VFSPath::Parse("/output").Value(), &root).Succeeded()); + Cook::CookDiagnostics diagnostics = {}; + Cook::CookLimits cook_limits = {.MaxWorkingBytes = ZKilo(256)}; + ZEngine::Managers::CookedPackageSnapshot package = {}; + ASSERT_TRUE(ZEngine::Managers::CookedPackageSnapshot::Capture(std::move(root), scene_id, schemas, cook_limits, *allocator, &diagnostics, &package).Succeeded()); + SlabAllocationRoute route = {}; + ASSERT_TRUE(route.Initialize(&cook_memory.MainArena, ZMega(1))); + const ZEngine::Managers::CookedCpuAssetLimits limits = {.MaxAssets = 4, .MaxObjectBytes = ZKilo(8), .MaxMeshWorkspaceBytes = ZKilo(64)}; + Cache cache = {}; + ASSERT_TRUE(cache.Initialize(route, schemas, limits, 1, 10, 5)); + struct CacheGuard + { + Cache& Owner; + ~CacheGuard() + { + while (Owner.PumpOne()) + { + } + EXPECT_TRUE(Owner.Shutdown()); + } + } cache_guard{cache}; + Cache::Lease lease = {}, cancel_lease = {}, material_lease = {}, mesh_lease = {}, environment_lease = {}; + ASSERT_TRUE(cache.Request(std::move(package), &lease)); + ASSERT_TRUE(lease.Retain(&cancel_lease)); + ASSERT_TRUE(lease.Retain(&material_lease)); + ASSERT_TRUE(lease.Retain(&mesh_lease)); + ASSERT_TRUE(lease.Retain(&environment_lease)); + ASSERT_TRUE(cache.PumpOne()); + Cache::Result material_assets = {}; + ASSERT_TRUE(material_lease.Poll(&material_assets)); + ASSERT_EQ(material_assets.StateValue, Cache::State::Ready); + ASSERT_NE(material_assets.Assets, nullptr); + Upload upload = {}; + ASSERT_TRUE(upload.Capture(std::move(lease), identity)); + std::array material_uploads = {&upload}; + ZEngine::Rendering::Materials::CookedMaterialVersion cooked_material = {}; + using DescriptorState = ZEngine::Rendering::SceneTextureDescriptorState; + EXPECT_EQ(ZEngine::Rendering::Materials::PrepareCookedMaterial(*material_assets.Assets, material_identity, material_uploads, *Manager, &cooked_material), DescriptorState::Pending); + EXPECT_FALSE(cooked_material); + EXPECT_EQ(ZEngine::Rendering::Materials::PrepareCookedMaterial(*material_assets.Assets, material_identity, {}, *Manager, &cooked_material), DescriptorState::Failed); + const std::array duplicate_uploads = {&upload, &upload}; + EXPECT_EQ(ZEngine::Rendering::Materials::PrepareCookedMaterial(*material_assets.Assets, material_identity, duplicate_uploads, *Manager, &cooked_material), DescriptorState::Failed); + EXPECT_EQ(upload.Digest(), digest); + Upload cancelled = {}; + ASSERT_TRUE(cancelled.Capture(std::move(cancel_lease), identity)); + const auto before_cancel = Device->GpuMem.GetAllocationStatistics(); + ASSERT_TRUE(cancelled.Prepare(*Manager)); + ASSERT_TRUE(cancelled.StageOne()); + cancelled.Reset(); + EXPECT_EQ(Device->GpuMem.GetAllocationStatistics().AdmittedBytes, before_cancel.AdmittedBytes); + EXPECT_EQ(upload.State(), Upload::Phase::SourceReady); + auto replaced_object = bmp; + replaced_object[54] = 200; + ASSERT_TRUE(backend.WriteFile(VFS::VFSPath::Parse(object_path.c_str()).Value(), {replaced_object.data(), replaced_object.size()}).Succeeded()); + ASSERT_TRUE(upload.Prepare(*Manager)); + bool staged = false; + std::thread worker([&] { staged = upload.StageOne(); }); + worker.join(); + ASSERT_TRUE(staged); + EXPECT_EQ(upload.State(), Upload::Phase::Staged); + EXPECT_EQ(upload.StagedBytes(), 4u); + + const auto incompatible = CreateProducerTestTexture("cooked-wrong-format", true, 16); + ASSERT_TRUE(incompatible.Valid()); + RRMUploadBatchTestHelper::InitializeTextureUploadSlots(*Manager, *Device, UploadCommands, false); + TextureUploadSlotsInitialized = true; + { + using Batch = ZEngine::Rendering::Textures::CookedSceneTextureBatch; + Cache::Lease batch_lease = {}; + ASSERT_TRUE(material_lease.Retain(&batch_lease)); + RRM::SceneResourcePreparationId batch_scene = {}; + ASSERT_TRUE(Manager->PrepareSceneResources({87, 1}, 1, {}, &batch_scene, RRM::ScenePreparationContent::MeshMaterials, nullptr, nullptr, false, material_assets.Assets)); + Batch batch = {}; + EXPECT_FALSE(batch.Initialize(std::move(batch_lease), route, *Manager, *Device, batch_scene, 0)); + EXPECT_TRUE(batch_lease); + ASSERT_TRUE(batch.Initialize(std::move(batch_lease), route, *Manager, *Device, batch_scene, 1)); + EXPECT_FALSE(batch_lease); + ASSERT_EQ(batch.Step(0, 0), Batch::State::Pending); + ASSERT_NE(batch.Find(identity), nullptr); + ASSERT_EQ(batch.Step(0, 0), Batch::State::Pending); + EXPECT_EQ(batch.Find(identity)->State(), Upload::Phase::Staged); + batch.Reset(); + ASSERT_TRUE(Manager->ReleaseSceneResourcePreparation(batch_scene)); + Manager->PollSceneResourcePreparations(); + RRMUploadBatchTestHelper::FlushTextureReleases(*Manager); + EXPECT_EQ(batch.Status(), Batch::State::Empty); + } + { + { + Cache::Lease invalid_target_lease = {}; + ASSERT_TRUE(environment_lease.Retain(&invalid_target_lease)); + ZEngine::Rendering::Textures::CookedEnvironmentUpload upload_environment = {}; + ASSERT_TRUE(upload_environment.Capture(std::move(invalid_target_lease), environment_identity)); + EXPECT_EQ(upload_environment.Digest(), environment_digest); + ASSERT_TRUE(upload_environment.Prepare(*Manager)); + ASSERT_TRUE(upload_environment.StageOne()); + EXPECT_EQ(upload_environment.StagedBytes(), 96u); + EXPECT_FALSE(upload_environment.Submit(0, 0, incompatible)); + EXPECT_EQ(upload_environment.State(), ZEngine::Rendering::Textures::CookedEnvironmentUpload::Phase::Staged); + } + RRM::SceneEnvironmentInput sky = {}; + sky.Sky.Mode = ZEngine::Rendering::Scenes::SkyMode::HDRI; + sky.Sky.EnvironmentMap = environment_id; + sky.SkyRevision = 1; + { + using EnvironmentBatch = ZEngine::Rendering::Textures::CookedSceneEnvironmentBatch; + Cache::Lease cancellation_lease = {}; + ASSERT_TRUE(environment_lease.Retain(&cancellation_lease)); + RRM::SceneResourcePreparationId cancellation_scene = {}; + ASSERT_TRUE(Manager->PrepareSceneResources({86, 1}, 1, {}, &cancellation_scene, RRM::ScenePreparationContent::MeshMaterials, &sky, nullptr, false, material_assets.Assets)); + EnvironmentBatch cancelled = {}; + EXPECT_FALSE(cancelled.Initialize(std::move(cancellation_lease), identity, *Manager, *Device, cancellation_scene)); + EXPECT_TRUE(cancellation_lease); + ASSERT_TRUE(cancelled.Initialize(std::move(cancellation_lease), environment_identity, *Manager, *Device, cancellation_scene)); + ASSERT_EQ(cancelled.Step(0, 0), EnvironmentBatch::State::Pending); + ASSERT_EQ(cancelled.Step(0, 0), EnvironmentBatch::State::Pending); + const auto cancelled_target = cancelled.Texture(); + ASSERT_TRUE(cancelled_target.Valid()); + cancelled.Reset(); + ASSERT_TRUE(Manager->ReleaseSceneResourcePreparation(cancellation_scene)); + Manager->PollSceneResourcePreparations(); + RRMUploadBatchTestHelper::FlushTextureReleases(*Manager); + EXPECT_FALSE(Manager->CanWriteTextureVersion(cancelled_target)); // Destruction is queued for device retirement. + } + RRM::SceneResourcePreparationId environment_scene = {}; + ASSERT_TRUE(Manager->PrepareSceneResources({88, 1}, 1, {}, &environment_scene, RRM::ScenePreparationContent::MeshMaterials, &sky, nullptr, false, material_assets.Assets)); + EXPECT_FALSE(Manager->FinishCookedSceneResources(environment_scene)); + using EnvironmentBatch = ZEngine::Rendering::Textures::CookedSceneEnvironmentBatch; + EnvironmentBatch environment = {}; + ASSERT_TRUE(environment.Initialize(std::move(environment_lease), environment_identity, *Manager, *Device, environment_scene, true)); + ASSERT_EQ(environment.Step(0, 0), EnvironmentBatch::State::Pending); + ASSERT_EQ(environment.Step(0, 0), EnvironmentBatch::State::Pending); + ASSERT_EQ(environment.Step(0, 0), EnvironmentBatch::State::Pending); + const auto cube_target = environment.Texture(); + ASSERT_TRUE(cube_target.Valid()); + Manager->SubmitAsyncUploads(); + const auto* cube_ticket = Manager->FindStreamingUploadTicket(cube_target); + ASSERT_NE(cube_ticket, nullptr); + ASSERT_NE(cube_ticket->CompletionTimeline, nullptr); + cube_ticket->CompletionTimeline->Wait(cube_ticket->CompletionValue); + RRMUploadBatchTestHelper::RetireTextureUploadSlots(*Manager); + EXPECT_EQ(Manager->GetTextureDecodeState(cube_target), RRM::TextureDecodeState::Succeeded); + ASSERT_EQ(environment.Step(0, 0), EnvironmentBatch::State::Ready); + auto readback = this->allocator().AllocateBuffer(96, VK_BUFFER_USAGE_TRANSFER_DST_BIT, GpuMemoryDomain::HostReadback, "cooked-environment-readback"); + ASSERT_TRUE(readback); + const auto* cube_texture = Device->GlobalTextures.Access(cube_target); + ASSERT_NE(cube_texture, nullptr); + const auto image = Device->ImageBufferManager.Access(cube_texture->BufferHandle)->GetHandle(); + const auto command = Commands[2]->GetHandle(); + VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + ASSERT_EQ(vkBeginCommandBuffer(command, &begin), VK_SUCCESS); + VkImageMemoryBarrier barrier = { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .srcAccessMask = VK_ACCESS_SHADER_READ_BIT, .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = image, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 6} + }; + vkCmdPipelineBarrier(command, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + const VkBufferImageCopy copy = { + .imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 6}, + .imageExtent = {1, 1, 1} + }; + vkCmdCopyImageToBuffer(command, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, readback.Handle, 1, ©); + barrier.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + barrier.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + VkBufferMemoryBarrier host_barrier = {.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, .dstAccessMask = VK_ACCESS_HOST_READ_BIT, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .buffer = readback.Handle, .offset = 0, .size = 96}; + vkCmdPipelineBarrier(command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 0, nullptr, 1, &host_barrier, 1, &barrier); + ASSERT_EQ(vkEndCommandBuffer(command), VK_SUCCESS); + VkSubmitInfo submit = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &command}; + const auto queue = Device->GetQueue(ZEngine::Rendering::QueueType::GRAPHIC_QUEUE).Handle; + ASSERT_EQ(vkQueueSubmit(queue, 1, &submit, VK_NULL_HANDLE), VK_SUCCESS); + ASSERT_EQ(vkQueueWaitIdle(queue), VK_SUCCESS); + std::array pixels = {}; + ASSERT_EQ(vmaCopyAllocationToMemory(this->allocator().Allocator, readback.Allocation, 0, pixels.data(), 96), VK_SUCCESS); + for (uint32_t i = 0; i < pixels.size(); ++i) + { + EXPECT_FLOAT_EQ(pixels[i], float(i + 1)); + } + this->allocator().FreeBuffer(readback); + ASSERT_TRUE(Manager->FinishCookedSceneResources(environment_scene)); + Manager->PollSceneResourcePreparations(); + RRM::SceneEnvironmentPreparation pending_environment = {}; + auto environment_reader = Manager->ClaimSceneEnvironmentPreparation(&pending_environment); + ASSERT_TRUE(environment_reader); + EXPECT_EQ(pending_environment.CookedSource, cube_target); + EXPECT_EQ(pending_environment.CookedFaceWidth, 1u); + EXPECT_EQ(pending_environment.CookedSourceHash, environment_header.SourceHash); + ZEngine::Rendering::Textures::TextureHandle taken_source = {}; + ASSERT_TRUE(Manager->TakeCookedSceneEnvironment(environment_scene, &taken_source)); + EXPECT_EQ(taken_source, cube_target); + ZEngine::Rendering::Textures::TextureHandle duplicate_source = {}; + EXPECT_FALSE(Manager->TakeCookedSceneEnvironment(environment_scene, &duplicate_source)); + environment_reader.Reset(); + ASSERT_TRUE(Manager->ReleaseSceneResourcePreparation(environment_scene)); + Manager->PollSceneResourcePreparations(); + Manager->ReleaseTexture(taken_source); + environment.Reset(); + } + EXPECT_FALSE(upload.Submit(0, 0, incompatible)); + EXPECT_EQ(upload.State(), Upload::Phase::Staged); + auto specification = Device->GlobalTextures.Access(incompatible)->Specification; + specification.Format = ZEngine::Rendering::Specifications::ImageFormat::R8G8B8A8_SRGB; + const auto target = Device->CreateTexture(specification, "cooked-texture"); + ASSERT_TRUE(target.Valid()); + struct DescriptorPreparation final : ZEngine::Rendering::ISceneTextureDescriptorPreparation + { + ZEngine::Rendering::SceneTextureDescriptorState State = ZEngine::Rendering::SceneTextureDescriptorState::Pending; + ZEngine::Rendering::Textures::TextureHandle Texture = {}; + ZEngine::Helpers::Handle Image = {}; + bool Current = false; + ZEngine::Rendering::SceneTextureDescriptorState PrepareSceneTextureDescriptor(ZEngine::Rendering::Textures::TextureHandle texture, ZEngine::Helpers::Handle image, uint64_t* publication) override + { + Texture = texture; + Image = image; + if (State == ZEngine::Rendering::SceneTextureDescriptorState::Ready) + { + *publication = 42; + } + return State; + } + bool IsSceneTextureDescriptorCurrent(ZEngine::Rendering::Textures::TextureHandle texture, ZEngine::Helpers::Handle image, uint64_t publication) override + { + return Current && texture == Texture && image == Image && publication == 42; + } + } descriptor; + struct DescriptorRestore + { + RRM& Manager; + ZEngine::Rendering::ISceneTextureDescriptorPreparation* Previous = nullptr; + ~DescriptorRestore() + { + RRMUploadBatchTestHelper::SetTextureDescriptorPreparation(Manager, Previous); + } + } restore{*Manager, RRMUploadBatchTestHelper::SetTextureDescriptorPreparation(*Manager, &descriptor)}; + Cache::Statistics leases_before_submit = {}; + ASSERT_TRUE(cache.TryGetStatistics(&leases_before_submit)); + RRMUploadBatchTestHelper::SaturateTextureUploadQueue(*Manager); + EXPECT_FALSE(upload.Submit(0, 0, target)); + EXPECT_EQ(upload.State(), Upload::Phase::Staged); + EXPECT_EQ(upload.SubmittedBinding(), nullptr); + Cache::Statistics leases_after_refusal = {}; + ASSERT_TRUE(cache.TryGetStatistics(&leases_after_refusal)); + EXPECT_EQ(leases_after_refusal.Leases, leases_before_submit.Leases); + RRMUploadBatchTestHelper::ClearTextureUploadJobs(*Manager); + ASSERT_TRUE(upload.Submit(0, 0, target)); + Cache::Statistics leases_after_submit = {}; + ASSERT_TRUE(cache.TryGetStatistics(&leases_after_submit)); + EXPECT_EQ(leases_after_submit.Leases + 1, leases_before_submit.Leases); + const auto* binding = upload.SubmittedBinding(); + ASSERT_NE(binding, nullptr); + EXPECT_EQ(binding->Identity, identity); + EXPECT_EQ(binding->ObjectDigest, digest); + EXPECT_EQ(binding->Handle, target); + EXPECT_EQ(Manager->FindStreamingUploadTicket(target), nullptr); // Not submitted to the GPU yet. + EXPECT_EQ(upload.Poll(), Upload::Readiness::Pending); + EXPECT_FALSE(Manager->IsTextureVersionPinned(target)); + EXPECT_EQ(upload.ProducerDescriptorBinding(), nullptr); + Manager->SubmitAsyncUploads(); + const auto* ticket = Manager->FindStreamingUploadTicket(target); + ASSERT_NE(ticket, nullptr); + ticket->CompletionTimeline->Wait(ticket->CompletionValue); + RRMUploadBatchTestHelper::RetireTextureUploadSlots(*Manager); + EXPECT_EQ(upload.Poll(), Upload::Readiness::Pending); // Descriptor not yet published. + EXPECT_TRUE(Manager->IsTextureVersionPinned(target)); + EXPECT_FALSE(Manager->CanWriteTextureVersion(target)); + descriptor.State = ZEngine::Rendering::SceneTextureDescriptorState::Ready; + descriptor.Current = true; + EXPECT_EQ(upload.Poll(), Upload::Readiness::AwaitingConsumer); + ASSERT_NE(upload.ProducerDescriptorBinding(), nullptr); + EXPECT_EQ(upload.ProducerDescriptorBinding()->ObjectDigest, digest); + auto retained = upload.RetainPreparedVersion(); + ASSERT_TRUE(retained); + ZEngine::Rendering::Meshes::MeshMaterial explicit_values = {}; + ASSERT_TRUE(ZEngine::Rendering::Materials::BuildCookedMaterial(*material_assets.Assets, material_identity, {upload.ProducerDescriptorBinding(), 1}, &explicit_values)); + const RRM::TextureVersionPin* explicit_pins[5] = {&retained, nullptr, nullptr, nullptr, nullptr}; + RRM::MaterialVersionPin explicit_result = {}; + EXPECT_EQ(Manager->PrepareExplicitMaterialVersion(explicit_values, explicit_pins, &explicit_result), DescriptorState::Pending); + EXPECT_FALSE(explicit_result); + auto wrong_values = explicit_values; + wrong_values.AlbedoMap = INVALID_MAP_HANDLE; + EXPECT_EQ(Manager->PrepareExplicitMaterialVersion(wrong_values, explicit_pins, &explicit_result), DescriptorState::Failed); + EXPECT_FALSE(explicit_result); + descriptor.Current = false; + EXPECT_EQ(upload.ProducerDescriptorBinding(), nullptr); + EXPECT_FALSE(upload.RetainPreparedVersion()); + descriptor.State = ZEngine::Rendering::SceneTextureDescriptorState::Pending; + EXPECT_EQ(upload.Poll(), Upload::Readiness::Pending); + descriptor.State = ZEngine::Rendering::SceneTextureDescriptorState::Ready; + descriptor.Current = true; + EXPECT_EQ(upload.Poll(), Upload::Readiness::AwaitingConsumer); + EXPECT_EQ(upload.ReadyBinding(), nullptr); + EXPECT_EQ(ZEngine::Rendering::Materials::PrepareCookedMaterial(*material_assets.Assets, material_identity, material_uploads, *Manager, &cooked_material), DescriptorState::Pending); + EXPECT_FALSE(cooked_material); + const uint8_t expected[4] = {30, 20, 10, 255}; + CheckTextureReadback(target, expected); + const auto acknowledged = *ticket; + ASSERT_TRUE(Manager->AcknowledgeStreamingUploadAcquire(acknowledged, Timeline.get(), 1, Device->GraphicFamilyIndex)); + EXPECT_EQ(upload.Poll(), Upload::Readiness::Ready); + ASSERT_NE(upload.ReadyBinding(), nullptr); + EXPECT_EQ(upload.ReadyBinding()->ObjectDigest, digest); + ASSERT_EQ(ZEngine::Rendering::Materials::PrepareCookedMaterial(*material_assets.Assets, material_identity, material_uploads, *Manager, &cooked_material), DescriptorState::Ready); + EXPECT_EQ(ZEngine::Rendering::Materials::PrepareCookedMaterial(*material_assets.Assets, material_identity, material_uploads, *Manager, &cooked_material), DescriptorState::Failed); + EXPECT_EQ(cooked_material.Identity(), material_identity); + EXPECT_EQ(cooked_material.Digest(), material_digest); + ASSERT_TRUE(cooked_material.Pin().Get()); + EXPECT_EQ(cooked_material.Pin().Get()->Material.AlbedoMap, target.Index); + EXPECT_FLOAT_EQ(cooked_material.Pin().Get()->Material.AlbedoColor.y, 0.5f); + EXPECT_EQ(cooked_material.Pin().Get()->Material.AlphaMode, ZEngine::Rendering::Materials::AlphaMode::Blend); + ASSERT_NE(cooked_material.Pin().GetTexture(0), nullptr); + EXPECT_EQ(cooked_material.Pin().GetTexture(0)->Handle, target); + { + TLSFSlab metadata = {}; + metadata.Init(&Memory.MainArena, ZKilo(16)); + auto vertices = this->allocator().AllocateBuffer(1024, VK_BUFFER_USAGE_TRANSFER_DST_BIT, GpuMemoryDomain::HostReadback, "cooked-scene-vertices"); + auto indices = this->allocator().AllocateBuffer(1024, VK_BUFFER_USAGE_TRANSFER_DST_BIT, GpuMemoryDomain::HostReadback, "cooked-scene-indices"); + RRMUploadBatchTestHelper::InitializeCookedGeometry(*Manager, Memory.MainArena, vertices, indices); + struct GeometryGuard + { + RRM& Manager; + GpuAllocator& Allocator; + TLSFSlab& Metadata; + BufferView& Vertices; + BufferView& Indices; + VkDevice Device; + ~GeometryGuard() + { + Manager.EndFrame(); + vkDeviceWaitIdle(Device); + RRMUploadBatchTestHelper::ReleaseSceneBuffersAfterIdle(Manager); + RRMUploadBatchTestHelper::ReleaseCookedGeometryAfterIdle(Manager); + Allocator.FreeBuffer(Vertices); + Allocator.FreeBuffer(Indices); + Metadata.Shutdown(); + } + } geometry_guard{*Manager, this->allocator(), metadata, vertices, indices, s_vk->Device}; + using MeshBatch = ZEngine::Rendering::Meshes::CookedSceneMeshBatch; + { + Cache::Lease cancellation_lease = {}; + ASSERT_TRUE(material_lease.Retain(&cancellation_lease)); + RRM::SceneResourcePreparationId cancellation_scene = {}; + ASSERT_TRUE(Manager->PrepareSceneResources({94, 1}, 1, std::span(&mesh_id, 1), &cancellation_scene, RRM::ScenePreparationContent::MeshMaterials, nullptr, nullptr, false, material_assets.Assets)); + MeshBatch cancelled_batch = {}; + const std::array duplicate_meshes = {mesh_id, mesh_id}; + EXPECT_FALSE(cancelled_batch.Initialize(std::move(cancellation_lease), duplicate_meshes, route, *Manager, metadata, cancellation_scene)); + EXPECT_TRUE(cancellation_lease); + ASSERT_TRUE(cancelled_batch.Initialize(std::move(cancellation_lease), std::span(&mesh_id, 1), route, *Manager, metadata, cancellation_scene)); + for (uint32_t i = 0; i < 4; ++i) + ASSERT_EQ(cancelled_batch.Step(0), MeshBatch::State::Pending); + cancelled_batch.Cancel(); + EXPECT_EQ(cancelled_batch.Status(), MeshBatch::State::Draining); + EXPECT_FALSE(cancelled_batch.Reset()); + Manager->EndFrame(); + ASSERT_EQ(vkDeviceWaitIdle(s_vk->Device), VK_SUCCESS); + ASSERT_EQ(cancelled_batch.Step(0), MeshBatch::State::Cancelled); + ASSERT_TRUE(cancelled_batch.Reset()); + Manager->BeginFrame(0); + ASSERT_TRUE(Manager->ReleaseSceneResourcePreparation(cancellation_scene)); + Manager->PollSceneResourcePreparations(); + } + { + // Model an older accepted batch's retained staging alongside new + // staging recorded by the generic batch path. + auto prior = Device->GpuMem.AllocateBuffer(64, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, GpuMemoryDomain::HostStaging, "prior-batch-staging"); + auto rejected = Device->GpuMem.AllocateBuffer(64, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, GpuMemoryDomain::HostStaging, "rejected-batch-staging"); + ASSERT_TRUE(prior); + ASSERT_TRUE(rejected); + ASSERT_EQ(RRMUploadBatchTestHelper::BatchStagingCount(*Manager, 0), 0u); + const auto prior_signal = RRMUploadBatchTestHelper::BatchLastSignal(*Manager, 0); + ASSERT_NE(prior_signal, 0u); + RRMUploadBatchTestHelper::AppendBatchStaging(*Manager, 0, prior); + RRMUploadBatchTestHelper::AppendBatchStaging(*Manager, 0, rejected); + RRMUploadBatchTestHelper::DiscardStagingTail(*Manager, 0, 1); + EXPECT_EQ(RRMUploadBatchTestHelper::BatchStagingCount(*Manager, 0), 1u); + EXPECT_EQ(RRMUploadBatchTestHelper::BatchStagingHandle(*Manager, 0, 0), prior.Handle); + EXPECT_EQ(RRMUploadBatchTestHelper::BatchLastSignal(*Manager, 0), prior_signal); + Manager->BeginFrame(0); + EXPECT_EQ(RRMUploadBatchTestHelper::BatchStagingCount(*Manager, 0), 0u); + } + { + Cache::Lease failure_lease = {}; + ASSERT_TRUE(material_lease.Retain(&failure_lease)); + RRM::SceneResourcePreparationId failed_scene = {}; + ASSERT_TRUE(Manager->PrepareSceneResources({94, 2}, 1, std::span(&mesh_id, 1), &failed_scene, RRM::ScenePreparationContent::MeshMaterials, nullptr, nullptr, false, material_assets.Assets)); + MeshBatch failed_batch = {}; + ASSERT_TRUE(failed_batch.Initialize(std::move(failure_lease), std::span(&mesh_id, 1), route, *Manager, metadata, failed_scene)); + for (uint32_t i = 0; i < 4; ++i) + ASSERT_EQ(failed_batch.Step(0), MeshBatch::State::Pending); + ASSERT_TRUE(failed_batch.HasRecordedUpload()); + ASSERT_EQ(RRMUploadBatchTestHelper::BatchStagingCount(*Manager, 0), 1u); + ASSERT_TRUE(RRMUploadBatchTestHelper::RejectRecordedCookedBatch(*Manager)); + EXPECT_EQ(RRMUploadBatchTestHelper::BatchStagingCount(*Manager, 0), 0u); + EXPECT_EQ(failed_batch.Step(0), MeshBatch::State::Draining); + EXPECT_FALSE(failed_batch.Reset()); + EXPECT_EQ(failed_batch.Step(0), MeshBatch::State::Failed); + EXPECT_TRUE(failed_batch.Reset()); + EXPECT_EQ(RRMUploadBatchTestHelper::OutstandingCookedMeshUploads(*Manager), 0u); + Manager->PollSceneResourcePreparations(); + EXPECT_TRUE(RRMUploadBatchTestHelper::BatchSubmissionFailed(*Manager)); + ASSERT_TRUE(Manager->ReleaseSceneResourcePreparation(failed_scene)); + Manager->PollSceneResourcePreparations(); + EXPECT_FALSE(RRMUploadBatchTestHelper::BatchSubmissionFailed(*Manager)); + } + ZEngine::Rendering::Scenes::MeshInstance instance = {}; + instance.MeshUUID = mesh_id; + instance.Transform(0, 3) = 7.f; + RRM::SceneInitialFrameInput initial = {}; + initial.Instances = {&instance, 1}; + RRM::SceneResourcePreparationId scene = {}; + ASSERT_TRUE(Manager->PrepareSceneResources({89, 1}, 1, std::span(&mesh_id, 1), &scene, RRM::ScenePreparationContent::MeshMaterials, nullptr, &initial, false, material_assets.Assets)); + MeshBatch batch = {}; + ASSERT_TRUE(batch.Initialize(std::move(mesh_lease), std::span(&mesh_id, 1), route, *Manager, metadata, scene)); + ASSERT_EQ(batch.Step(0), MeshBatch::State::Pending); + ASSERT_EQ(batch.Step(0), MeshBatch::State::Pending); + ASSERT_EQ(batch.Step(0), MeshBatch::State::Pending); + ASSERT_EQ(batch.Step(0), MeshBatch::State::Pending); + Manager->EndFrame(); + ASSERT_EQ(vkDeviceWaitIdle(s_vk->Device), VK_SUCCESS); + ASSERT_EQ(batch.Step(0), MeshBatch::State::Ready); + ASSERT_TRUE(batch.Reset()); + ASSERT_TRUE(Manager->AppendCookedSceneMaterial(scene, *material_assets.Assets, cooked_material)); + ASSERT_TRUE(Manager->FinishCookedSceneResources(scene)); + const auto ready = FinishScenePreparation(scene); + ASSERT_EQ(ready.State, RRM::SceneResourcePreparationState::ResourcesRetained); + EXPECT_TRUE(ready.Cooked); + EXPECT_EQ(ready.InitialDrawCount, 1u); + EXPECT_TRUE(ready.InitialBuffersPrepared); + // A later candidate's rejected batch must not invalidate an already + // retained outgoing scene when the device itself remains healthy. + RRMUploadBatchTestHelper::SetInjectedBatchRejection(*Manager); + Manager->PollSceneResourcePreparations(); + RRM::SceneResourcePreparationResult still_ready = {}; + ASSERT_TRUE(Manager->PollSceneResourcePreparation(scene, &still_ready)); + EXPECT_EQ(still_ready.State, RRM::SceneResourcePreparationState::ResourcesRetained); + EXPECT_FALSE(RRMUploadBatchTestHelper::BatchSubmissionFailed(*Manager)); + ZEngine::Rendering::Scenes::PreparedSceneFrame prepared_frame = {}; + ASSERT_TRUE(Manager->GetPreparedSceneFrame(scene, &prepared_frame)); + ASSERT_EQ(prepared_frame.Draws.size(), 1u); + EXPECT_TRUE(prepared_frame.Draws[0].Transparent); + ASSERT_EQ(prepared_frame.Transforms.size(), 1u); + EXPECT_FLOAT_EQ(prepared_frame.Transforms[0](0, 3), 7.f); + RRM::MeshGeometrySnapshot geometry = {}; + RRM::MaterialVersionSnapshot material = {}; + ASSERT_TRUE(Manager->GetPreparedSceneGeometry(scene, mesh_id, &geometry)); + ASSERT_TRUE(Manager->GetPreparedSceneMaterial(scene, material_id, &material)); + EXPECT_NE(geometry.Version, 0u); + EXPECT_EQ(material.Material.AlphaMode, ZEngine::Rendering::Materials::AlphaMode::Blend); + const auto geometry_version = geometry.Version; + auto scene_reader = Manager->AcquireSceneResourceLease(scene, 2); + ASSERT_TRUE(scene_reader); + Manager->NotifyPreparedSceneFrameRecorded(scene); + EXPECT_FALSE(Manager->GetPreparedSceneFrame(scene, &prepared_frame)); + ASSERT_TRUE(Manager->GetPreparedSceneGeometry(scene, mesh_id, &geometry)); + ASSERT_TRUE(Manager->GetPreparedSceneMaterial(scene, material_id, &material)); + EXPECT_EQ(geometry.Version, geometry_version); + EXPECT_EQ(material.Material.AlbedoMap, target.Index); + scene_reader.Reset(); + ASSERT_TRUE(Manager->ReleaseSceneResourcePreparation(scene)); + RRM::SceneResourcePreparationResult retiring = {}; + for (uint32_t i = 0; i < 100 && Manager->PollSceneResourcePreparation(scene, &retiring); ++i) + { + Manager->PollSceneResourcePreparations(); + } + EXPECT_FALSE(Manager->PollSceneResourcePreparation(scene, &retiring)); + } + { + TLSFSlab metadata = {}; + metadata.Init(&Memory.MainArena, ZKilo(16)); + auto vertices = this->allocator().AllocateBuffer(1024, VK_BUFFER_USAGE_TRANSFER_DST_BIT, GpuMemoryDomain::HostReadback, "cooked-driver-vertices"); + auto indices = this->allocator().AllocateBuffer(1024, VK_BUFFER_USAGE_TRANSFER_DST_BIT, GpuMemoryDomain::HostReadback, "cooked-driver-indices"); + RRMUploadBatchTestHelper::InitializeCookedGeometry(*Manager, Memory.MainArena, vertices, indices); + struct DriverGeometryGuard + { + RRM& Manager; + GpuAllocator& Allocator; + TLSFSlab& Metadata; + BufferView& Vertices; + BufferView& Indices; + VkDevice Device; + ~DriverGeometryGuard() + { + Manager.EndFrame(); + vkDeviceWaitIdle(Device); + RRMUploadBatchTestHelper::ReleaseSceneBuffersAfterIdle(Manager); + RRMUploadBatchTestHelper::ReleaseCookedGeometryAfterIdle(Manager); + Allocator.FreeBuffer(Vertices); + Allocator.FreeBuffer(Indices); + Metadata.Shutdown(); + } + } geometry_guard{*Manager, this->allocator(), metadata, vertices, indices, s_vk->Device}; + ZEngine::Rendering::Scenes::MeshInstance instance = {}; + instance.MeshUUID = mesh_id; + RRM::SceneInitialFrameInput initial = {}; + initial.Instances = {&instance, 1}; + RRM::SceneEnvironmentInput sky = {}; + sky.Sky.Mode = ZEngine::Rendering::Scenes::SkyMode::HDRI; + sky.Sky.EnvironmentMap = environment_id; + sky.SkyRevision = 1; + RRM::SceneResourcePreparationId scene = {}; + ASSERT_TRUE(Manager->PrepareSceneResources({95, 1}, 1, std::span(&mesh_id, 1), &scene, RRM::ScenePreparationContent::MeshMaterials, &sky, &initial, false, material_assets.Assets)); + using Mailbox = ZEngine::Rendering::CookedSceneGpuMailbox; + Mailbox mailbox = {}; + Mailbox::Ticket request = {}; + ASSERT_TRUE(mailbox.Begin(material_lease, std::span(&mesh_id, 1), environment_identity, route, metadata, *Manager, *Device, scene, 1, &request)); + Mailbox::Ticket duplicate_request = {}; + EXPECT_FALSE(mailbox.Begin(material_lease, std::span(&mesh_id, 1), environment_identity, route, metadata, *Manager, *Device, scene, 1, &duplicate_request)); + EXPECT_FALSE(duplicate_request); + ZEngine::Rendering::Primitives::Semaphore acquired(Device.get(), true); + bool acknowledged_batch = false; + Mailbox::Outcome outcome = Mailbox::Outcome::Pending; + for (uint32_t i = 0; i < 100 && outcome == Mailbox::Outcome::Pending; ++i) + { + ASSERT_TRUE(mailbox.PumpOne(0, 0)); + ASSERT_TRUE(mailbox.Poll(request, &outcome)); + Manager->EndFrame(); + Manager->SubmitAsyncUploads(); + ASSERT_EQ(vkDeviceWaitIdle(s_vk->Device), VK_SUCCESS); + RRMUploadBatchTestHelper::RetireTextureUploadSlots(*Manager); + Manager->BeginFrame(0); + auto* texture = mailbox.FindTexture(request, identity); + const auto* binding = texture ? texture->SubmittedBinding() : nullptr; + const auto* ticket = binding ? Manager->FindStreamingUploadTicket(binding->Handle) : nullptr; + if (!acknowledged_batch && ticket) + { + const auto accepted = *ticket; + const auto* resource = Device->GlobalTextures.Access(binding->Handle); + ASSERT_NE(resource, nullptr); + const auto image = Device->ImageBufferManager.Access(resource->BufferHandle)->GetHandle(); + const auto command = Commands[2]->GetHandle(); + ASSERT_EQ(vkResetCommandBuffer(command, 0), VK_SUCCESS); + VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + ASSERT_EQ(vkBeginCommandBuffer(command, &begin), VK_SUCCESS); + VkImageMemoryBarrier barrier = { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, .dstAccessMask = VK_ACCESS_SHADER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = image, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1} + }; + vkCmdPipelineBarrier(command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + ASSERT_EQ(vkEndCommandBuffer(command), VK_SUCCESS); + const uint64_t signal = 1; + const auto semaphore = acquired.GetHandle(); + VkTimelineSemaphoreSubmitInfo timeline = {.sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO, .signalSemaphoreValueCount = 1, .pSignalSemaphoreValues = &signal}; + VkSubmitInfo submit = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .pNext = &timeline, .commandBufferCount = 1, .pCommandBuffers = &command, .signalSemaphoreCount = 1, .pSignalSemaphores = &semaphore}; + ASSERT_EQ(vkQueueSubmit(Device->GetQueue(ZEngine::Rendering::QueueType::GRAPHIC_QUEUE).Handle, 1, &submit, VK_NULL_HANDLE), VK_SUCCESS); + acquired.Wait(signal); + ASSERT_TRUE(Manager->AcknowledgeStreamingUploadAcquire(accepted, &acquired, signal, Device->GraphicFamilyIndex)); + acknowledged_batch = true; + } + } + ASSERT_EQ(outcome, Mailbox::Outcome::Submitted); + EXPECT_TRUE(acknowledged_batch); + EXPECT_FALSE(mailbox.Cancel(request)); + ASSERT_TRUE(mailbox.Drained()); + ASSERT_TRUE(mailbox.Acknowledge(request)); + EXPECT_FALSE(mailbox.Poll(request, &outcome)); + RRM::SceneEnvironmentPreparation environment = {}; + RRM::SceneResourceLease environment_reader = {}; + for (uint32_t i = 0; i < 100 && !environment_reader; ++i) + { + Manager->PollSceneResourcePreparations(); + environment_reader = Manager->ClaimSceneEnvironmentPreparation(&environment); + } + ASSERT_TRUE(environment_reader); + ZEngine::Rendering::Textures::TextureHandle source = {}; + ASSERT_TRUE(Manager->TakeCookedSceneEnvironment(scene, &source)); + ASSERT_TRUE(Manager->CompleteSceneEnvironmentPreparation(scene, true)); // The sky bake is exercised separately. + RRM::SceneResourcePreparationResult ready = {}; + ASSERT_TRUE(Manager->PollSceneResourcePreparation(scene, &ready)); + ASSERT_EQ(ready.State, RRM::SceneResourcePreparationState::ResourcesRetained); + EXPECT_EQ(ready.InitialDrawCount, 1u); + ZEngine::Rendering::Scenes::PreparedSceneFrame frame = {}; + ASSERT_TRUE(Manager->GetPreparedSceneFrame(scene, &frame)); + ASSERT_EQ(frame.Draws.size(), 1u); + EXPECT_TRUE(frame.Draws[0].Transparent); + environment_reader.Reset(); + ASSERT_TRUE(Manager->ReleaseSceneResourcePreparation(scene)); + for (uint32_t i = 0; i < 100 && Manager->PollSceneResourcePreparation(scene, &ready); ++i) + Manager->PollSceneResourcePreparations(); + EXPECT_FALSE(Manager->PollSceneResourcePreparation(scene, &ready)); + Manager->ReleaseTexture(source); + RRMUploadBatchTestHelper::FlushTextureReleases(*Manager); + RRM::SceneResourcePreparationId cancelled_scene = {}; + ASSERT_TRUE(Manager->PrepareSceneResources({96, 1}, 1, std::span(&mesh_id, 1), &cancelled_scene, RRM::ScenePreparationContent::MeshMaterials, &sky, &initial, false, material_assets.Assets)); + Mailbox::Ticket cancelled = {}; + ASSERT_EQ(mailbox.TryBegin(material_lease, std::span(&mesh_id, 1), environment_identity, route, metadata, *Manager, *Device, cancelled_scene, 1, &cancelled, true), Mailbox::Admission::Queued); + EXPECT_NE(cancelled.Serial, request.Serial); + EXPECT_FALSE(mailbox.PumpOne(0, 0, true)); // No new uploads without a valid frame. + auto gate = std::make_unique(Device.get(), true); + for (uint32_t i = 0; i < 11 && !mailbox.MeshUploadRecorded(cancelled); ++i) + { + ASSERT_TRUE(mailbox.PumpOne(0, 0)); + if (mailbox.MeshUploadRecorded(cancelled)) + { + Swapchain.RenderTimeline = gate.get(); + Swapchain.RenderTimelineNextValue = 1; + Manager->EndFrame(); // Accepted upload waits on the unsignalled timeline. + break; + } + Manager->EndFrame(); + Manager->SubmitAsyncUploads(); + ASSERT_EQ(vkDeviceWaitIdle(s_vk->Device), VK_SUCCESS); + RRMUploadBatchTestHelper::RetireTextureUploadSlots(*Manager); + Manager->BeginFrame(0); + } + // Always release the GPU gate before any fatal assertion or teardown. + EXPECT_TRUE(mailbox.MeshUploadRecorded(cancelled)); + EXPECT_GT(RRMUploadBatchTestHelper::OutstandingCookedMeshUploads(*Manager), 0u); + EXPECT_TRUE(mailbox.Cancel(cancelled)); + outcome = Mailbox::Outcome::Pending; + EXPECT_TRUE(mailbox.PumpOne(0, 0, true)); + EXPECT_TRUE(mailbox.Poll(cancelled, &outcome)); + EXPECT_EQ(outcome, Mailbox::Outcome::Pending); + gate->Signal(1); + EXPECT_EQ(vkDeviceWaitIdle(s_vk->Device), VK_SUCCESS); + Swapchain.RenderTimeline = nullptr; + Swapchain.RenderTimelineNextValue = 0; + for (uint32_t i = 0; i < 100 && outcome == Mailbox::Outcome::Pending; ++i) + { + ASSERT_TRUE(mailbox.PumpOne(0, 0, true)); // No new frame is begun during cleanup. + ASSERT_TRUE(mailbox.Poll(cancelled, &outcome)); + } + EXPECT_EQ(outcome, Mailbox::Outcome::Cancelled); + EXPECT_TRUE(mailbox.Acknowledge(cancelled)); + Manager->PollSceneResourcePreparations(); + EXPECT_FALSE(Manager->PollSceneResourcePreparation(cancelled_scene, &ready)); + RRMUploadBatchTestHelper::FlushTextureReleases(*Manager); + RRM::SceneResourcePreparationId shutdown_scene = {}; + ASSERT_TRUE(Manager->PrepareSceneResources({97, 1}, 1, std::span(&mesh_id, 1), &shutdown_scene, RRM::ScenePreparationContent::MeshMaterials, &sky, &initial, false, material_assets.Assets)); + Mailbox::Ticket shutdown_request = {}; + ASSERT_EQ(mailbox.TryBegin(material_lease, std::span(&mesh_id, 1), environment_identity, route, metadata, *Manager, *Device, shutdown_scene, 1, &shutdown_request, true), Mailbox::Admission::Queued); + for (uint32_t i = 0; i < 11; ++i) + { + ASSERT_TRUE(mailbox.PumpOne(0, 0)); + Manager->EndFrame(); + Manager->SubmitAsyncUploads(); + ASSERT_EQ(vkDeviceWaitIdle(s_vk->Device), VK_SUCCESS); + RRMUploadBatchTestHelper::RetireTextureUploadSlots(*Manager); + Manager->BeginFrame(0); + } + EXPECT_GT(RRMUploadBatchTestHelper::OutstandingCookedMeshUploads(*Manager), 0u); + EXPECT_TRUE(mailbox.MeshUploadRecorded(shutdown_request)); + mailbox.Seal(); + ASSERT_TRUE(mailbox.DrainAfterDeviceIdle()); + ASSERT_TRUE(mailbox.Poll(shutdown_request, &outcome)); + EXPECT_EQ(outcome, Mailbox::Outcome::Cancelled); + ASSERT_TRUE(mailbox.Acknowledge(shutdown_request)); + Manager->PollSceneResourcePreparations(); + EXPECT_FALSE(Manager->PollSceneResourcePreparation(shutdown_scene, &ready)); + Mailbox::Ticket refused = {}; + EXPECT_FALSE(mailbox.Begin(material_lease, std::span(&mesh_id, 1), environment_identity, route, metadata, *Manager, *Device, cancelled_scene, 1, &refused)); + EXPECT_FALSE(refused); + EXPECT_TRUE(mailbox.Drained()); + descriptor.Texture = target; + descriptor.Image = Device->GlobalTextures.Access(target)->BufferHandle; + } + auto moved_material = std::move(cooked_material); + EXPECT_FALSE(cooked_material); + EXPECT_EQ(moved_material.Identity(), material_identity); + EXPECT_EQ(moved_material.Digest(), material_digest); + cooked_material = std::move(moved_material); + EXPECT_FALSE(moved_material); + RRM::SceneResourcePreparationId incomplete_scene = {}; + const auto missing_mesh = *uuids::uuid::from_string("00000000-0000-4000-8000-000000000004"); + ASSERT_TRUE(Manager->PrepareSceneResources({90, 1}, 1, std::span(&missing_mesh, 1), &incomplete_scene, RRM::ScenePreparationContent::MeshMaterials, nullptr, nullptr, false, material_assets.Assets)); + EXPECT_FALSE(Manager->FinishCookedSceneResources(incomplete_scene)); + RRM::CookedMeshVersion empty_mesh = {}; + EXPECT_FALSE(Manager->AppendCookedSceneMesh(incomplete_scene, *material_assets.Assets, empty_mesh)); + ASSERT_TRUE(Manager->AppendCookedSceneMaterial(incomplete_scene, *material_assets.Assets, cooked_material)); + ASSERT_TRUE(Manager->ReleaseSceneResourcePreparation(incomplete_scene)); + Manager->PollSceneResourcePreparations(); + RRM::SceneResourcePreparationResult cancelled_scene = {}; + EXPECT_FALSE(Manager->PollSceneResourcePreparation(incomplete_scene, &cancelled_scene)); + RRM::SceneResourcePreparationId cooked_scene = {}; + ASSERT_TRUE(Manager->PrepareSceneResources({91, 1}, 1, {}, &cooked_scene, RRM::ScenePreparationContent::MeshMaterials, nullptr, nullptr, false, material_assets.Assets)); + ASSERT_TRUE(Manager->AppendCookedSceneMaterial(cooked_scene, *material_assets.Assets, cooked_material)); + EXPECT_FALSE(Manager->AppendCookedSceneMaterial(cooked_scene, *material_assets.Assets, cooked_material)); + ASSERT_TRUE(Manager->FinishCookedSceneResources(cooked_scene)); + EXPECT_EQ(FinishScenePreparation(cooked_scene).State, RRM::SceneResourcePreparationState::ResourcesRetained); + ASSERT_TRUE(Manager->ReleaseSceneResourcePreparation(cooked_scene)); + Manager->PollSceneResourcePreparations(); + RRM::SceneResourcePreparationResult retired_scene = {}; + EXPECT_FALSE(Manager->PollSceneResourcePreparation(cooked_scene, &retired_scene)); + RRM::SceneResourcePreparationId texture_owner = {}; + ASSERT_TRUE(Manager->PrepareSceneResources({92, 1}, 1, {}, &texture_owner, RRM::ScenePreparationContent::MeshMaterials, nullptr, nullptr, false, material_assets.Assets)); + ASSERT_TRUE(Manager->AppendCookedSceneTexture(texture_owner, *material_assets.Assets, upload)); + EXPECT_FALSE(Manager->AppendCookedSceneTexture(texture_owner, *material_assets.Assets, upload)); + ASSERT_TRUE(Manager->FinishCookedSceneResources(texture_owner)); + EXPECT_EQ(FinishScenePreparation(texture_owner).State, RRM::SceneResourcePreparationState::ResourcesRetained); + { + using Batch = ZEngine::Rendering::Textures::CookedSceneTextureBatch; + Cache::Lease batch_lease = {}; + ASSERT_TRUE(material_lease.Retain(&batch_lease)); + RRM::SceneResourcePreparationId batch_scene = {}; + ASSERT_TRUE(Manager->PrepareSceneResources({93, 1}, 1, {}, &batch_scene, RRM::ScenePreparationContent::MeshMaterials, nullptr, nullptr, false, material_assets.Assets)); + Batch batch = {}; + ASSERT_TRUE(batch.Initialize(std::move(batch_lease), route, *Manager, *Device, batch_scene, 1)); + ASSERT_EQ(batch.Step(0, 0), Batch::State::Pending); + ASSERT_EQ(batch.Step(0, 0), Batch::State::Pending); + ASSERT_EQ(batch.Step(0, 0), Batch::State::Pending); + auto* staged_texture = batch.Find(identity); + ASSERT_NE(staged_texture, nullptr); + ASSERT_NE(staged_texture->SubmittedBinding(), nullptr); + const auto batch_target = staged_texture->SubmittedBinding()->Handle; + Manager->SubmitAsyncUploads(); + const auto* batch_ticket = Manager->FindStreamingUploadTicket(batch_target); + ASSERT_NE(batch_ticket, nullptr); + const auto accepted = *batch_ticket; + accepted.CompletionTimeline->Wait(accepted.CompletionValue); + RRMUploadBatchTestHelper::RetireTextureUploadSlots(*Manager); + EXPECT_EQ(batch.Step(0, 0), Batch::State::Pending); + EXPECT_EQ(batch.StepMaterials(), Batch::State::Pending); + EXPECT_EQ(staged_texture->Poll(), Upload::Readiness::AwaitingConsumer); + const auto* batch_texture = Device->GlobalTextures.Access(batch_target); + ASSERT_NE(batch_texture, nullptr); + const auto image = Device->ImageBufferManager.Access(batch_texture->BufferHandle)->GetHandle(); + const auto command = Commands[2]->GetHandle(); + ASSERT_EQ(vkResetCommandBuffer(command, 0), VK_SUCCESS); + VkCommandBufferBeginInfo begin = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + ASSERT_EQ(vkBeginCommandBuffer(command, &begin), VK_SUCCESS); + VkImageMemoryBarrier barrier = { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, .dstAccessMask = VK_ACCESS_SHADER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = image, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1} + }; + vkCmdPipelineBarrier(command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + ASSERT_EQ(vkEndCommandBuffer(command), VK_SUCCESS); + ZEngine::Rendering::Primitives::Semaphore acquire(Device.get(), true); + const uint64_t signal = 1; + const auto semaphore = acquire.GetHandle(); + VkTimelineSemaphoreSubmitInfo timeline_info = {.sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO, .signalSemaphoreValueCount = 1, .pSignalSemaphoreValues = &signal}; + VkSubmitInfo submission = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .pNext = &timeline_info, .commandBufferCount = 1, .pCommandBuffers = &command, .signalSemaphoreCount = 1, .pSignalSemaphores = &semaphore}; + ASSERT_EQ(vkQueueSubmit(Device->GetQueue(ZEngine::Rendering::QueueType::GRAPHIC_QUEUE).Handle, 1, &submission, VK_NULL_HANDLE), VK_SUCCESS); + acquire.Wait(signal); + ASSERT_TRUE(Manager->AcknowledgeStreamingUploadAcquire(accepted, &acquire, signal, Device->GraphicFamilyIndex)); + ASSERT_EQ(batch.Step(0, 0), Batch::State::Ready); + auto material_state = Batch::State::Pending; + for (uint32_t i = 0; i <= material_assets.Assets->Size() && material_state == Batch::State::Pending; ++i) + { + material_state = batch.StepMaterials(); + } + ASSERT_EQ(material_state, Batch::State::Ready); + ASSERT_TRUE(Manager->FinishCookedSceneResources(batch_scene)); + EXPECT_EQ(FinishScenePreparation(batch_scene).State, RRM::SceneResourcePreparationState::ResourcesRetained); + batch.Reset(); + EXPECT_TRUE(Manager->IsTextureVersionPinned(batch_target)); + ASSERT_TRUE(Manager->ReleaseSceneResourcePreparation(batch_scene)); + Manager->PollSceneResourcePreparations(); + EXPECT_FALSE(Manager->IsTextureVersionPinned(batch_target)); + RRMUploadBatchTestHelper::FlushTextureReleases(*Manager); + } + descriptor.Current = false; + uint32_t material_index = 0; + RRM::MaterialVersionSnapshot material_snapshot = {}; + EXPECT_FALSE(Manager->GetFrameMaterialVersion(cooked_material.Pin(), &material_index, &material_snapshot)); + descriptor.Current = true; + Cache::Lease retained_closure = {}; + ASSERT_TRUE(material_lease.Retain(&retained_closure)); + material_lease.Reset(); + descriptor.State = ZEngine::Rendering::SceneTextureDescriptorState::Failed; + EXPECT_EQ(upload.Poll(), Upload::Readiness::Failed); + EXPECT_EQ(upload.State(), Upload::Phase::Failed); + EXPECT_EQ(upload.ProducerDescriptorBinding(), nullptr); + EXPECT_FALSE(upload.RetainPreparedVersion()); + descriptor.State = ZEngine::Rendering::SceneTextureDescriptorState::Ready; + VFS::VFSReadRootLease rejected_root = {}; + ASSERT_TRUE(context.AcquireReadRoot(VFS::VFSPath::Parse("/output").Value(), &rejected_root).Succeeded()); + ZEngine::Managers::CookedPackageSnapshot rejected_package = {}; + ASSERT_TRUE(ZEngine::Managers::CookedPackageSnapshot::Capture(std::move(rejected_root), scene_id, schemas, cook_limits, *allocator, &diagnostics, &rejected_package).Succeeded()); + Cache::Lease rejected_lease = {}; + ASSERT_TRUE(cache.Request(std::move(rejected_package), &rejected_lease)); + rejected_package.Reset(); + Upload rejected = {}; + ASSERT_TRUE(rejected.Capture(std::move(rejected_lease), identity)); + retained_closure.Reset(); + ASSERT_TRUE(rejected.Prepare(*Manager)); + ASSERT_TRUE(rejected.StageOne()); + const auto rejected_target = Device->CreateTexture(specification, "cooked-rejected-texture"); + ASSERT_TRUE(rejected_target.Valid()); + ASSERT_TRUE(rejected.Submit(0, 0, rejected_target)); + RRMUploadBatchTestHelper::ClearTextureUploadJobs(*Manager); + EXPECT_EQ(Manager->GetTextureDecodeState(rejected_target), RRM::TextureDecodeState::Failed); + EXPECT_EQ(rejected.Poll(), Upload::Readiness::Failed); + EXPECT_EQ(rejected.SubmittedBinding(), nullptr); + EXPECT_FALSE(Manager->IsTextureVersionPinned(rejected_target)); + rejected.Reset(); + upload.Reset(); + EXPECT_TRUE(Manager->IsTextureVersionPinned(target)); + retained.Reset(); + EXPECT_TRUE(Manager->IsTextureVersionPinned(target)); + cooked_material.Reset(); + EXPECT_TRUE(Manager->IsTextureVersionPinned(target)); + ASSERT_TRUE(Manager->ReleaseSceneResourcePreparation(texture_owner)); + Manager->PollSceneResourcePreparations(); + EXPECT_FALSE(Manager->IsTextureVersionPinned(target)); + ASSERT_TRUE(cache.PumpOne()); + ASSERT_TRUE(cache.Shutdown()); +} + +TEST_F(RenderResourceManagerBatchTest, TransferTextureStagingRefusalPreservesStateAndAllowsRetry) +{ + CheckTextureStagingAdmission(true); +} + TEST_F(RenderResourceManagerBatchTest, RejectedTransferTextureJobsReclaimSlotsAndStagingBeforeRetry) { CheckRejectedTextureUploadReclamation(true); diff --git a/ZEngine/tests/Rendering/ImageCodec_test.cpp b/ZEngine/tests/Rendering/ImageCodec_test.cpp index 174bf6295..96d7c0869 100644 --- a/ZEngine/tests/Rendering/ImageCodec_test.cpp +++ b/ZEngine/tests/Rendering/ImageCodec_test.cpp @@ -481,6 +481,43 @@ namespace ZEngine::Importers::ImageCodec EXPECT_EQ(route.LiveBytes, 0u); } + TEST(ImageCodecTest, RuntimeDecodeRetainsBottomUpPixelsAndPreservesThemOnFailure) + { + constexpr uint8_t tga[] = {0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 32, 0x28, 0, 0, 255, 255, 0, 255, 0, 255}; + BorrowedImageSource source = {}; + source.Bytes = {tga, sizeof(tga)}; + CountingImageRoute route = {}; + DecodedImage image = {}; + ASSERT_TRUE(DecodeEncodedImage(source.View(), route, {}, &image).Succeeded()); + ASSERT_EQ(image.Pixels().size(), 8u); + const uint8_t expected[] = {0, 255, 0, 255, 255, 0, 0, 255}; + EXPECT_TRUE(std::equal(image.Pixels().begin(), image.Pixels().end(), expected)); + EXPECT_EQ(route.LiveBlocks, 1u); + EXPECT_EQ(DecodeEncodedImage(source.View(), route, {.MaxDimension = 1}, &image).Error(), Core::VFS::VFSError::SizeLimitExceeded); + EXPECT_TRUE(std::equal(image.Pixels().begin(), image.Pixels().end(), expected)); + DecodedImage moved = std::move(image); + EXPECT_TRUE(image.Pixels().empty()); + EXPECT_EQ(moved.Pixels().size(), 8u); + moved.Reset(); + EXPECT_EQ(route.LiveBlocks, 0u); + } + + TEST(ImageCodecTest, RuntimeDecodeConvertsSixteenBitPngToRgba8) + { + constexpr uint8_t png[] = {0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x10, 0x06, 0x00, 0x00, 0x00, 0x4f, 0x85, 0x18, 0xca, 0x00, 0x00, 0x00, 0x11, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9c, 0x63, 0x10, 0x32, 0x09, 0xab, 0x98, 0xb5, 0xe7, 0xde, 0x07, 0x00, 0x0d, 0xa9, 0x04, 0x39, 0x53, 0x00, 0xc9, 0xba, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82}; + BorrowedImageSource source = {}; + source.Bytes = {png, sizeof(png)}; + CountingImageRoute route = {}; + DecodedImage image = {}; + ASSERT_TRUE(DecodeEncodedImage(source.View(), route, {}, &image).Succeeded()); + EXPECT_EQ(image.Description().BitsPerChannel, 16u); + ASSERT_EQ(image.Pixels().size(), 4u); + const uint8_t expected[] = {0x12, 0x56, 0x9a, 0xde}; + EXPECT_TRUE(std::equal(image.Pixels().begin(), image.Pixels().end(), expected)); + image.Reset(); + EXPECT_EQ(route.LiveBlocks, 0u); + } + TEST(ImageCodecTest, TGADetectionAndDimensionPreflightDoNotAllocateSpeculativeJPEGState) { constexpr uint8_t tga[] = {0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 32, 0x28, 11, 42, 93, 255, 11, 42, 93, 255}; diff --git a/ZEngine/tests/Rendering/RenderGraphTest.cpp b/ZEngine/tests/Rendering/RenderGraphTest.cpp index fd83b2b68..25df03a5e 100644 --- a/ZEngine/tests/Rendering/RenderGraphTest.cpp +++ b/ZEngine/tests/Rendering/RenderGraphTest.cpp @@ -2508,6 +2508,41 @@ TEST(RenderGraphQueueScheduleTest, CompiledComputeBackendPreservesCallbackQueueP manager.Shutdown(); } +TEST(RenderGraphQueueScheduleTest, BindlessAndTicketedImageConsumersUseGraphicsFamily) +{ + MemoryManager manager = {}; + manager.Initialize(16384, {}); + auto& arena = manager.MainArena; + auto device = std::make_unique(); + device->Arena = &arena; + device->HasSeparateComputeQueue = true; + device->HasSeparateTransferQueue = true; + RenderGraph graph = {}; + graph.Device = device.get(); + graph.Passes.init(&arena, 3); + graph.Resources.init(&arena, 1); + graph.Resources.push({}); + graph.Resources[0].HasStreamingTicket = true; + graph.SortedPassIndices.init(&arena, 3); + graph.QueueBatches.init(&arena, 3); + graph.QueueDependencies.init(&arena, 3); + graph.QueueOwnershipTransfers.init(&arena, 3); + TestComputePass compute = {}; + for (uint32_t index = 0; index < 3; ++index) + { + graph.Passes.push(MakePass(&arena, "Compute consumer")); + graph.Passes[index].Callback = &compute; + graph.SortedPassIndices.push(index); + } + graph.Passes[0].ReadsBindless = true; + AddRead(graph.Passes[1], 0); + RenderGraphTestAccess::BuildQueueSchedule(graph); + EXPECT_EQ(graph.Passes[0].Queue, Rendering::QueueType::GRAPHIC_QUEUE); + EXPECT_EQ(graph.Passes[1].Queue, Rendering::QueueType::GRAPHIC_QUEUE); + EXPECT_EQ(graph.Passes[2].Queue, Rendering::QueueType::COMPUTE_QUEUE); + manager.Shutdown(); +} + TEST(RenderGraphQueueScheduleTest, GroupsContiguousPassesByResolvedQueue) { MemoryManager manager{}; diff --git a/ZEngine/tests/Rendering/RenderResourceManagerTest.cpp b/ZEngine/tests/Rendering/RenderResourceManagerTest.cpp index e618354b8..74af47aa8 100644 --- a/ZEngine/tests/Rendering/RenderResourceManagerTest.cpp +++ b/ZEngine/tests/Rendering/RenderResourceManagerTest.cpp @@ -29,6 +29,34 @@ using namespace ZEngine::Core::VFS; using namespace ZEngine::Rendering; using namespace ZEngine::Managers; +TEST(TextureUploadSizeTest, ChecksBaseMipLayersDepthFormatAndOverflow) +{ + Textures::Texture texture = {}; + texture.Width = 4; + texture.Height = 2; + texture.BytePerPixel = 4; + texture.Specification.Format = Specifications::ImageFormat::R8G8B8A8_UNORM; + texture.Specification.Depth = 2; + texture.Specification.LayerCount = 3; + texture.BufferSize = 200; // Includes additional mip storage. + EXPECT_FALSE(texture.ContainsBaseMip(191)); + EXPECT_TRUE(texture.ContainsBaseMip(192)); + VkDeviceSize copy_bytes = 0; + EXPECT_TRUE(texture.ContainsBaseMip(200, ©_bytes)); + EXPECT_EQ(copy_bytes, 192u); + EXPECT_FALSE(texture.ContainsBaseMip(201)); + texture.BytePerPixel = 1; + EXPECT_FALSE(texture.ContainsBaseMip(192)); + texture.BytePerPixel = 4; + texture.Specification.Format = Specifications::ImageFormat::DEPTH16_UNORM; + EXPECT_FALSE(texture.ContainsBaseMip(192)); + texture.Specification.Format = Specifications::ImageFormat::R8G8B8A8_UNORM; + texture.Width = UINT32_MAX; + texture.Height = UINT32_MAX; + texture.BufferSize = std::numeric_limits::max(); + EXPECT_FALSE(texture.ContainsBaseMip(texture.BufferSize)); +} + struct RRMDecodeTestHelper { static uint32_t FrameMaterialCount(const RenderResourceManager& manager) @@ -688,9 +716,15 @@ class ResourceVersionDecodeTest : public ResourceVersionTest TEST_F(ResourceVersionDecodeTest, PinnedDecodeDeferralRetriesButDestructionReclaimsItsPixelsAndLease) { - const auto texture = MakeTexture(); - Device->GlobalTextures.Access(texture)->BufferSize = 4; - auto pin = Manager->PinTextureVersion(texture); + const auto texture = MakeTexture(); + auto* resource = Device->GlobalTextures.Access(texture); + resource->Width = resource->Height = 1; + resource->BytePerPixel = 4; + resource->Specification.Depth = 1; + resource->Specification.LayerCount = 1; + resource->BufferSize = 4; + ASSERT_TRUE(resource->ContainsBaseMip(4)); + auto pin = Manager->PinTextureVersion(texture); ASSERT_TRUE(pin); ASSERT_TRUE(RRMDecodeTestHelper::Track(*Manager, texture)); auto deferral = RRMDecodeTestHelper::MakeDeferral(*Manager, texture); @@ -718,6 +752,24 @@ TEST_F(ResourceVersionDecodeTest, PinnedDecodeDeferralRetriesButDestructionRecla EXPECT_TRUE(Device->TextureHandleToDispose.empty()); } +TEST_F(ResourceVersionDecodeTest, ShortDecodedDeferralFailsAndReleasesItsLease) +{ + const auto texture = MakeTexture(); + auto* resource = Device->GlobalTextures.Access(texture); + resource->BytePerPixel = 4; + resource->Specification.Depth = 1; + resource->Specification.LayerCount = 1; + resource->BufferSize = 16 * 8 * 4; + ASSERT_TRUE(RRMDecodeTestHelper::Track(*Manager, texture)); + auto deferral = RRMDecodeTestHelper::MakeDeferral(*Manager, texture); + ASSERT_NE(deferral.Pixels, nullptr); + EXPECT_TRUE(RRMDecodeTestHelper::ProcessDeferral(*Manager, deferral)); + EXPECT_EQ(deferral.Pixels, nullptr); + EXPECT_EQ(RRMDecodeTestHelper::ActiveDecodeSlabs(*Manager), 0u); + EXPECT_EQ(Manager->GetTextureDecodeState(texture), RenderResourceManager::TextureDecodeState::Failed); + EXPECT_TRUE(Manager->CanWriteTextureVersion(texture)); +} + TEST_F(ResourceVersionDecodeTest, LateDecodeCannotModifyTheReusedTextureSlotOrKeepItsSlab) { const auto texture = MakeTexture(); diff --git a/ZEngine/tests/Rendering/SceneSubmissionProgress_test.cpp b/ZEngine/tests/Rendering/SceneSubmissionProgress_test.cpp index b20d39bed..d8b26289d 100644 --- a/ZEngine/tests/Rendering/SceneSubmissionProgress_test.cpp +++ b/ZEngine/tests/Rendering/SceneSubmissionProgress_test.cpp @@ -14,13 +14,14 @@ TEST(SceneSubmissionProgressTest, StartsEmptyAndCountsOnlyExplicitAcceptedNotifi EXPECT_EQ(snapshot.Count, 0u); EXPECT_FALSE(progress.TryRead(nullptr)); progress.Record(0, 10); - progress.Record(4, 20, 8192); - progress.Record(4, 30, 8192); + progress.Record(4, 20, 8192, 13); + progress.Record(4, 30, 8192, 14); ASSERT_TRUE(progress.TryRead(&snapshot)); EXPECT_EQ(snapshot.Generation, 4u); EXPECT_EQ(snapshot.Count, 2u); EXPECT_EQ(snapshot.FirstTimestampNs, 20u); EXPECT_EQ(snapshot.DrawCount, 8192u); + EXPECT_EQ(snapshot.FirstTimelineValue, 13u); progress.Record(3, 40); ASSERT_TRUE(progress.TryRead(&snapshot)); EXPECT_EQ(snapshot.Count, 2u); @@ -31,6 +32,7 @@ TEST(SceneSubmissionProgressTest, StartsEmptyAndCountsOnlyExplicitAcceptedNotifi EXPECT_EQ(snapshot.Count, 1u); EXPECT_EQ(snapshot.FirstTimestampNs, 50u); EXPECT_EQ(snapshot.DrawCount, 0u); + EXPECT_EQ(snapshot.FirstTimelineValue, 0u); } TEST(SceneSubmissionProgressTest, ConcurrentReadsNeverMixGenerationCountAndFirstTimestamp) @@ -40,8 +42,8 @@ TEST(SceneSubmissionProgressTest, ConcurrentReadsNeverMixGenerationCountAndFirst std::thread writer([&] { for (uint64_t generation = 1; generation <= 100000; ++generation) { - progress.Record(generation, generation * 11, static_cast(generation)); - progress.Record(generation, generation * 11 + 1, static_cast(generation)); + progress.Record(generation, generation * 11, static_cast(generation), generation * 3); + progress.Record(generation, generation * 11 + 1, static_cast(generation), generation * 3 + 1); } finished.store(true, std::memory_order_release); }); @@ -55,6 +57,7 @@ TEST(SceneSubmissionProgressTest, ConcurrentReadsNeverMixGenerationCountAndFirst coherent &= snapshot.Generation >= last_generation && snapshot.FirstTimestampNs == snapshot.Generation * 11; coherent &= snapshot.Generation ? (snapshot.Count == 1 || snapshot.Count == 2) : snapshot.Count == 0; coherent &= snapshot.DrawCount == snapshot.Generation; + coherent &= snapshot.FirstTimelineValue == snapshot.Generation * 3; last_generation = snapshot.Generation; } } while (!finished.load(std::memory_order_acquire)); diff --git a/ZEngine/tests/VFS/AssetRegistryTest.cpp b/ZEngine/tests/VFS/AssetRegistryTest.cpp index dff2ce63d..843b91dba 100644 --- a/ZEngine/tests/VFS/AssetRegistryTest.cpp +++ b/ZEngine/tests/VFS/AssetRegistryTest.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -68,6 +69,25 @@ TEST_F(AssetRegistryTest, UUIDLookupReturnsCorrectRecord) EXPECT_STREQ(rec->Path.CStr(), "/project/mesh.glb"); } +TEST_F(AssetRegistryTest, SourceIndependenceAuditDeniesRegistryReadsAndImports) +{ + const auto id = MakeUUID(); + Reg(m_registry, id, Managers::AssetType::MESH, "/project/mesh.glb"); + AssetRecordSnapshot snapshot = {}; + ASSERT_TRUE(m_registry.TryReadRecord(id, &snapshot)); + m_registry.EnableLookupDenialAudit(); + EXPECT_EQ(m_registry.FindByUUID(id), nullptr); + EXPECT_EQ(m_registry.FindByPath(P("/project/mesh.glb")), nullptr); + EXPECT_FALSE(m_registry.TryReadRecord(id, &snapshot)); + EXPECT_EQ(snapshot.Record.UUID, id); // denied reads preserve the caller's output + EXPECT_EQ(m_registry.DeniedLookupCount(), 3u); + + ZEngine::Importers::ImportCoordinator imports = {}; + imports.EnableImportDenialAudit(); + EXPECT_TRUE(imports.Enqueue(P("/project/mesh.glb")).is_nil()); + EXPECT_EQ(imports.DeniedImportCount(), 1u); +} + TEST_F(AssetRegistryTest, OneAssetTypeCanUseTheEntireDeclaredCapacityWithoutGrowingTheSharedArena) { const auto cursor = m_manager.MainArena.m_current_offset; diff --git a/ZEngine/tests/VFS/vfs_mount_test.cpp b/ZEngine/tests/VFS/vfs_mount_test.cpp index 8780e1978..d54485ad0 100644 --- a/ZEngine/tests/VFS/vfs_mount_test.cpp +++ b/ZEngine/tests/VFS/vfs_mount_test.cpp @@ -221,6 +221,30 @@ TEST_F(VFSContextTest, OpenUnmountedPathFailsNotFound) EXPECT_EQ(r.Error(), VFSError::NotFound); } +TEST_F(VFSContextTest, EngineOnlyAuditDeniesSourceReadsAndCountsAttempts) +{ + MockBackend project("project"); + MockBackend engine("engine"); + ASSERT_TRUE(m_ctx.Mount(&project, P("/project"), 0).Succeeded()); + ASSERT_TRUE(m_ctx.Mount(&engine, P("/ZodiacEngine"), 0).Succeeded()); + m_ctx.EnableEngineOnlyAudit(); + const auto source = P("/project/mesh.glb"); + EXPECT_EQ(m_ctx.Open(source, VFSOpenFlags::Read).Error(), VFSError::PermissionDenied); + EXPECT_EQ(m_ctx.Stat(source).Error(), VFSError::PermissionDenied); + EXPECT_EQ(m_ctx.Exists(source).Error(), VFSError::PermissionDenied); + EXPECT_EQ(m_ctx.List(P("/project"), &m_manager.MainArena).Error(), VFSError::PermissionDenied); + VFSReadRootLease read_root = {}; + VFSPublicationRootLease publication = {}; + VFSScratchRootLease scratch = {}; + EXPECT_EQ(m_ctx.AcquireReadRoot(P("/project"), &read_root).Error(), VFSError::PermissionDenied); + EXPECT_EQ(m_ctx.AcquirePublicationRoot(P("/project"), &publication).Error(), VFSError::PermissionDenied); + EXPECT_EQ(m_ctx.AcquireScratchRoot(P("/project"), &scratch).Error(), VFSError::PermissionDenied); + EXPECT_EQ(m_ctx.DeniedSourceAccesses(), 7u); + (void) m_ctx.Open(P("/ZodiacEngine/Settings/test"), VFSOpenFlags::Read); + EXPECT_EQ(m_ctx.DeniedSourceAccesses(), 7u); + EXPECT_EQ(engine.LastOpen, P("/Settings/test")); +} + TEST_F(VFSContextTest, WriteOpsRequireWritableBackend) { MockBackend read_only("ro", VFSBackendCaps::Read); diff --git a/docs/design/future-plan/profiling.md b/docs/design/future-plan/profiling.md index dd6fdbc94..cc305012c 100644 --- a/docs/design/future-plan/profiling.md +++ b/docs/design/future-plan/profiling.md @@ -1,29 +1,686 @@ # ZEngine — Profiling and Instrumentation -**Status:** CPU instrumentation facilities, arena tracking, and render-graph timestamp collection exist. Frame integration, presentation, and validation coverage are incomplete. +**Status:** Extensive proposal for review; not implemented or a production-readiness +claim. Revised 2026-11-10 against the M1 source tree. +**Scope:** in-editor Performance tooling, CPU/GPU instrumentation and correlation, +memory attribution, optional Tracy integration, bounded capture/export and qualification. +**Related:** [PR #872](https://github.com/JeanPhilippeKernel/RendererEngine/pull/872), +[atomic-layout task #875](https://github.com/JeanPhilippeKernel/RendererEngine/issues/875), +[M2 #873](https://github.com/JeanPhilippeKernel/RendererEngine/issues/873) and +[M3 #874](https://github.com/JeanPhilippeKernel/RendererEngine/issues/874). -## Current facilities +All new defaults, APIs and stages below are proposals. Existing facilities are +distinguished from planned behavior. Benchmark/build history belongs in implementation +PR artifacts, not in this contract. -`Profiling.h` selects one of three compile-time paths: +## 1. Recommendation and outcomes -| Condition | Scope/value/alloc behavior | +Deliver an integrated **Performance panel** for everyday diagnosis, backed by shared +engine instrumentation, and **Tracy** for deep trace analysis. Keep the existing Memory +Profiler as a compatible view of shared memory telemetry. Do not build a second full +Tracy implementation inside the editor or require a Tracy connection to inspect frame +times and engine budgets. + +The intended workflow is: + +1. Notice a frame spike, resource growth or budget pressure inside the editor. +2. Select the affected frame/request and inspect CPU owners, GPU passes and memory. +3. Capture a bounded interval with explicit completeness and instrumentation mode. +4. Use Tracy or external hardware tools for deeper investigation when needed. +5. Compare against a matched baseline and attach reproducible evidence to a PR. + +Observed duration is not causal attribution. A long present call, idle GPU interval, +cache-miss count or increasing RSS does not by itself identify a responsible subsystem. + +| Consumer | Purpose | Does not replace | +|---|---|---| +| In-editor Performance panel | Live summaries, bounded frame inspection and capture controls | Arbitrarily deep trace navigation or hardware analysis | +| Memory Profiler panel | Owner/pool/resource budgets, counts and trends | General process heap profiler or physical-residency authority | +| Optional runtime overlay | Minimal development-host frame/memory indicators | Editor tooling or shipping telemetry policy | +| Tracy desktop application | Detailed thread zones, task flows, supported GPU/allocation analysis | Engine-authoritative budget and lease accounting | +| Headless capture/test helpers | Repeatable machine-readable measurements | Interactive UI or hardware profiler | +| Platform/vendor tools | Sampling, scheduler, coherency/cache, shader counters and display investigation | Portable engine instrumentation | + +Initial non-goals: remote shipping-game control, player analytics, automatic +false-sharing diagnosis, shader instruction profiling, whole-system tracing, universal +heap interception, crash-safe trace recovery, and exact physical CPU/GPU residency. +Each needs a separately reviewed capability rather than an implied checkbox. + +## 2. Verified starting point and gaps + +| Facility | Present in source | Gap to this proposal | +|---|---|---| +| `Profiling.h` | Tracy, lightweight fallback or no-op macro paths | Current mutually exclusive routing cannot supply both local detail and Tracy | +| CPU fallback | `ScopeGuard`, counters, 4,096-entry sample/value arrays | No normal-loop frame advance/presentation or thread/parent identity in old samples | +| Capture | 262,144 records, prefix/timed capture, CSV and drop count | Start/stop/export require quiescent writers; not live editor capture controls | +| Thread CPU clock | Validity-aware thread clock and wall/non-CPU comparison | Not universal platform support or scheduling attribution | +| Tracy | OFF by default, pinned client v0.13.1, on-demand configuration | Incomplete GPU/frame/allocation integration and session workflow | +| Memory profiler | Named arenas, current/sample peaks and warnings; main-thread update | Not whole-process live payload or allocation-origin tracking | +| Allocation hooks | Macros exist | Fallback `RecordAlloc`/`RecordFree` are empty; allocator coverage incomplete | +| CPU resource ledgers | TLSF live/high-water, cache/upload ownership statistics | Not a unified attributed investigation view | +| GPU timing | RenderGraph query pools and completed `RGPassTiming` records | No integrated timing UI or complete common-clock submission/presentation trace | +| GPU memory | VMA/driver budgets, allocation ledger, environment/transient statistics | Distinct domains still need provenance and detailed ownership views | +| Editor UI | `MemoryProfilerPanel` | No full Performance panel or capture/comparison workflow | + +Currently `ZENGINE_PROFILING` is supplied for Debug and RelWithDebInfo, not ordinary +Release. `ZENGINE_TRACY` defaults OFF and its engine compile definition is private; +audit propagation to editor/helpers before claiming consistent instrumentation. +Section 11 proposes build policy; those are not existing additional options. + +Two correctness prerequisites cannot be solved by merely wiring old APIs: + +- Resetting a shared writer index and copying a CPU sample array can race workers + still writing reserved entries. Atomic reservation is not completed publication. + Replace that transport before enabling normal-loop snapshots; do not expose live + producer arrays to ZUI. +- Names are currently borrowed pointers. A formatted stack name, such as an arena + plot label, cannot remain borrowed until later CSV export. Register stable names + or copy bounded dynamic data into owned metadata. + +Legacy CSV remains a separate format. Do not reinterpret old records as containing +thread identity, calibrated queues or actual display timestamps never captured. + +## 3. Architecture and ownership + +```text + Main / render / worker threads Renderer / allocator owners + zones, tasks and counters GPU/query and memory records + | | + +------ stable instrument IDs ----+ + | + Engine Profiling service + +---------------+----------------+ + | | + bounded local transport optional Tracy adapter + per-thread publication supported client APIs + | | + one collector Tracy client/server + / | \ | + immutable UI bounded test detailed analysis + snapshots archive summaries + | | + Performance / +--> async export --> versioned artifact + Memory / overlay +``` + +Shared instrumentation has distinct sinks; this does not require translation into +Tracy's private wire protocol. Tracy GPU integration may need its own bounded query +contexts under the renderer ownership rules in section 7. + +### 3.1. Placement + +- `ZEngine/ZEngine/Profiling`: configuration, stable metadata/IDs, CPU transport, + snapshots, capture state, export schema and Tracy adapters. Core facilities must + not depend on Tetragrama, ZUI, an active scene or a GPU. +- Engine/application/render pipeline and worker scheduler: phase/frame/task markers + and lifetime hooks; not ad-hoc profiler file output. +- Renderer/RenderGraph: GPU capabilities, queries, completion-gated results and + resource snapshots under renderer-owned lifetime. +- Memory/asset/upload owners: authoritative counters plus optional detail hooks; + profiler failures never change allocation or retirement semantics. +- `Tetragrama/Panels`: Performance presentation and existing Memory panel, issuing + bounded service commands instead of accessing raw buffers. +- `Scripts` and tests: artifact validation/comparison and process helpers, without + machine-specific probes as implicit dependencies. + +API names beyond the existing tree are implementation choices. Use one engine +service lifetime, not one global singleton per UI or backend. + +### 3.2. Thread and storage contract + +| Operation | Owner / rule | +|---|---| +| Configuration/thread/name registration | Bootstrap/control owner; bounded registration before hot use | +| CPU emission | Registered current thread, sole writer of its ring | +| Drain/history/capture mutation | One collector, off main/render critical paths | +| GPU queries/results | Renderer owner, genuine completion ownership | +| Allocator statistics | Existing synchronization; no added global heap lock | +| UI commands | Main-thread bounded command channel, acknowledged results | +| UI drawing | Immutable retained snapshot, never mutable producer storage | +| File/compression/export work | Bounded export worker, not the producer/collector | + +Preallocate per-thread rings. Release-publish fully initialized records and acquire- +consume published records only. Never overwrite unread slots. Thread exit seals a +watermark and transfers storage ownership; registration generations prevent OS thread-ID +reuse from aliasing old lanes. Excess/unregistered threads continue engine work while +their detail is explicitly unavailable/dropped. + +Prefer completed CPU spans containing start/duration and parent/span IDs, with a bounded +per-thread scope stack, rather than unmatched transport begin/end pairs. Scope/nesting +loss invalidates derived exclusive time. A thread-local recursion guard excludes +profiler internals; profiler memory is separately owned and charged. + +Publish snapshots through a bounded ownership mailbox: Writing → Ready → Reading → Free. +A slow panel pins its slot; the collector skips/coalesces publication instead of +blocking or overwriting it. No C++ data race is permitted behind a seqlock-like retry. +Closing/reopening/detaching panels releases readers safely. + +## 4. Data, identity and measurement semantics + +### 4.1. Record envelope + +Use a versioned bounded binary schema with tagged payload and availability/quality +flags. Initial capacity arithmetic assumes 64-byte transport records; larger logical +events use bounded linked records counted against the same budget. Final packing is +reviewed before implementation, not inferred from the list below. + +Session metadata includes build/commit/instrumentation version, OS/backend/device/driver, +clock units/domains, mode/caps/categories, profiler overhead configuration, and applicable +fixture/shader identities. Site/category/thread IDs refer to session-owned dictionaries. +Names are bounded and immutable; dynamic values are owned bounded copies, not borrowed +`cstring` pointers. Dictionary exhaustion records unknown-name/loss status. + +Events carry applicable identities, not every identity on every record: + +- session/configuration epoch and thread registration generation; +- monotonic CPU timestamp, category/site, span/parent or async-operation ID; +- main iteration, render frame, immutable snapshot sequence and scene generation; +- queue/device epoch, accepted submission ID and completion association; +- allocation/resource ID plus reuse generation, request ID or typed asset version. + +Raw pointers/process-local ECS IDs are not durable exported identity; asset UUID alone +does not identify a version. Use explicit links, not nearest-timestamp matching. + +### 4.2. Timing definitions + +| Quantity | Meaning / limitation | +|---|---| +| CPU wall zone | Includes blocking and descheduling inside the instrumented interval | +| Thread CPU time | Calling-thread execution accounting, only with a valid supported clock | +| Wall minus CPU | Non-CPU interval including waits, descheduling and clock overhead; not causal proof | +| Main work / interval | Work phase versus elapsed spacing of main iterations | +| Render work / interval | Recording/submission work versus spacing of render iterations | +| Task timing | Enqueue → start, start → finish and dependency waits are distinct | +| GPU pass | Timestamp-bracketed interval on a valid queue; can include stalls/dependencies | +| Submission latency | Accepted CPU submit through observed GPU completion; not display time | +| Presentation | Acquire wait, present request and actual display feedback are separate observations | + +Main/render rates can differ. Define `MainFrame` and `RenderFrame` plus snapshot links, +not a shared mutable frame counter reset by competing threads. Preserve cross-frame +span boundaries; any view clipping is labelled. Represent skipped/minimized frames, +resize/device epochs, failed submissions and absent presentation explicitly. + +Use `duration_cast` on a validated monotonic clock; do not assume native +`steady_clock::duration::count()` is nanoseconds. Record units/frequency/domain. Invalid +time is N/A, not zero. Bytes/counts stay integers through storage/export, with floating +conversion only where needed for drawing. + +Inclusive/exclusive aggregation requires complete nesting on one thread. Do not subtract +async children or sum parallel workers as elapsed frame time. Percentiles name their +population, interval, mode, losses and warm-up policy. Incomplete data is flagged, +not silently removed to improve a result. + +## 5. CPU instrumentation + +Required initial coverage: + +- Main/render boundaries and input, application/ECS update, command flush, hierarchy/ + scene preparation, immutable handoff, recording and submission phases. +- Worker enqueue/start/finish with stable async flow IDs and distinct wait/useful-work + spans. Ordinary nested scopes cannot migrate across threads. +- Asset I/O/decode/import/upload preparation, pipelines/resource admission, retaining + compatibility with existing `SceneLoad.*`, thumbnail and content-browser sites. +- Explicit waits/lock sites only on shortlisted hotspots, not hooks on every mutex. +- Optional long-task thread CPU samples with unavailable status; process-wide/coarse + CPU accounting must not masquerade as a precise thread clock. + +Keep macro compatibility where practical. Register stable sites once and fan out to +enabled sinks without per-scope formatting/allocation. Distinguish same-named functions +using site/module identity. Dynamic labels require owned metadata rather than reuse of +literal-only APIs. + +The local UI supports bounded thread lanes, nesting and hot-region aggregates. Arbitrary +long-session search, symbolized sampled stacks and OS context switches remain Tracy/ +platform-tool depth. Sampling/call stacks/lock traces are opt-in separately; surface +symbol/privilege/support requirements and overhead. A checkbox cannot claim an unbuilt +or unqualified platform capability. + +## 6. In-editor Performance and Memory experience + +### 6.1. Proposed layout + +```text + Performance [Live/Pause view] [Arm/Record/Stop] [Export] [Tracy status] + Mode: Summary Capture: Idle GPU: delayed 3 frames Loss: 0 + ------------------------------------------------------------------ + CPU work/interval | GPU queue spans | Present feedback | Memory + ----------------- bounded recent-frame graphs -------------------- + Selected: Render 1824 / Main 1841 / Snapshot 1900 / Scene generation 7 + [Overview] [CPU] [GPU] [Memory] [Capture health] + ------------------------------------------------------------------ + CPU: thread lanes/hot regions GPU: queue/pass rows/dependencies + Memory: owners/resources Health: caps, loss, unavailable data +``` + +This wireframe defines relationships, not final styling. Reuse ZUI docking/theme/input +and accessibility conventions; support keyboard focus, high DPI and narrow widths. +Units/scope/status labels are mandatory, and color is not the only status indicator. + +### 6.2. Behavior + +- Overview separates main/render work and intervals, supported GPU summaries and + distinct CPU/GPU budgets. It does not present a misleading universal bottleneck badge. +- Select a spike to inspect correlated records with GPU age/pending state. Pause freezes + the **view**, not gameplay/capture. A pinned view cannot block producers; expired + detail is reported rather than retained indefinitely. +- CPU shows lanes/nesting, inclusive/exclusive values and task links, marking loss or + incomplete hierarchy. Wall waits are never labelled CPU execution. +- GPU shows pass names, resolved queues, completion age and submission links. Pass sums + are not GPU frame time; uncalibrated clock lanes remain distinct. +- Memory shares providers with the existing panel, with owner/resource drill-down; + the UI never resamples mutable allocator/renderer internals unsafely. +- Health always exposes mode/categories, coverage start, age, query/ring/dictionary + capacity, losses and export failures. Unsupported data is N/A. +- Commands are acknowledged nonblocking operations. Disabled controls show why. + Closing a panel does not implicitly terminate an explicitly armed/recording session. +- Initial summary history: 512 frames with virtualized bounded detail. This is not a + promise of complete events for all 512 frames; show actual retained ranges/evictions. + +### 6.3. Runtime and headless hosts + +A minimal opt-in runtime overlay has low-rate summaries/health, no editor dependency +and no input interception without host consent. Headless hosts use the same service +through tested CLI/control seams. Introduce exact CLI syntax with implementation; +do not advertise fictional existing flags. No shipping network service is implied. + +## 7. GPU profiling and Tracy GPU integration + +### 7.1. Renderer safety + +Preserve RenderGraph's valid-bit/period checks, finite per-frame query capacity and +host-query-reset policy until a separately validated replacement exists. Check each +resolved queue family; graphics timestamps do not imply compute/transfer support. + +Associate timing sets with original frame/context, queue and device epoch. Read after +existing real completion ownership permits, or with nonblocking availability checks. +No added `WAIT_BIT`, fence wait, queue-idle or device-idle to make a chart current. +Delayed/failed/missing data is explicit, never invented zero-duration work. + +Skipped/aborted passes and failed submissions must not appear completed. Pool/context +retirement follows genuine last-use completion through resize, cancellation, shutdown +and device loss. Query overflow omits instrumented passes with a count; it must not +affect rendering correctness or create an unbounded query pool. + +### 7.2. Correlation and clocks + +Portable baseline: per-pass durations with logical CPU submission/task links. +Cross-queue subtraction or CPU/GPU offset estimation is prohibited without a validated +common/calibrated domain. Where `VK_EXT_calibrated_timestamps` or a qualified equivalent +exists, retain clock pairs, domains, age and maximum deviation; refresh at bounded +cadence and invalidate discontinuities/device changes. Display alignment uncertainty. +Without support, separate lanes with logical links are a valid degraded mode. + +Present request, acquire wait, accepted submit and actual display time require separate +capabilities. Without display feedback do not claim input-to-photon/display latency. +MoltenVK/native diagnostics require backend-specific evidence; temporary M1 Metal probes +are not permanent portable engine infrastructure. + +### 7.3. Tracy adapter + +Audit the pinned client's documented Vulkan contexts/zones/collection APIs. Do not +inject stored `RGPassTiming` into an invented Tracy wire format. Define ownership for +each actual queue and primary/secondary worker-recording arrangement. + +Prefer shared engine timing policy/sites. If supported APIs cannot share query lifetimes, +permit separate local and Tracy query pools only in explicit Detailed+Tracy mode; +charge both capacities/overhead and show dual instrumentation. Summary must not silently +double queries. Claim GPU integration only after multi-queue, delayed collection, +secondary recording, resize and context-destruction tests pass. + +GPU intervals include scheduling/dependencies, not isolated shader throughput. Use +vendor/native tools for shader/hardware counters and label perturbation. Validation +layers prove neither timing accuracy nor absence of GPU bottlenecks. + +## 8. Memory accounting and attribution + +### 8.1. Distinct quantities + +| Domain | Expose | Never claim | +|---|---|---| +| Named arenas | Reserved/known committed bytes, offset and sampled high-water | Offset equals live payload or RSS | +| Slabs/pools | Backing/control, live charged bytes/blocks, operation high-water, available requested bytes | Backing and suballocation usage are independent physical RAM | +| Assets/scenes | Version pins, active/candidate/retiring ownership, cache retention | UUID alone identifies one immutable allocation/version | +| Upload/readback | Pool/staging, live leases/bytes, completion retention | Submitted means reusable | +| VMA | Allocation bytes/count versus blocks | Allocations and blocks are additive usage | +| Driver heaps | Usage/budget with driver/fallback provenance and age | Exact process-exclusive VRAM/residency | +| RenderGraph | Virtual resources, physical backing and aliasing | Virtual and physical are additive | +| OS process | RSS/working set and supported footprint/commit, with definitions | Full owner ledger or exact logical reclamation | + +Unified memory can appear in both CPU/GPU views. Do not publish a combined RAM+VRAM +total as physical consumption. Unknown commitment/residency is unknown, not zero. +Frame-sampled arena peaks may miss within-frame maxima; distinguish them from +per-operation high-water ledgers. + +### 8.2. Attribution levels + +1. **Summary:** authoritative owner/pool/domain counts, budgets and high-water; no + allocation stack or full resource scan every frame. +2. **Tracked detail:** bounded lifetime records, allocation ID/reuse generation, + requested/charged bytes, owner/category and logical backing relationship, with + resource names/asset versions when available. +3. **Stack detail:** optional filtered/sampled CPU allocation stacks, bounded symbols + and explicit coverage/overhead. + +Hook reviewed engine-controlled OS backing, slab/pool and resource boundaries. Failed +realloc preserves its old live record; pointer reuse receives a new generation. +Cross-thread frees preserve identity/causal ordering. Arena reset retires logical +allocations by owner/reset epoch, not fabricated native frees. Relate backing and +suballocation records to prevent double counting. + +Do not put arena backing plus every bump allocation into one unqualified Tracy heap. +Use supported named pools/arenas or labelled logical events, with safe reset/reconnect +semantics. Exclude profiler recursion. Third-party/driver/library routes not instrumented +remain explicitly outside coverage. + +Late attach cannot recover unrecorded allocation origins. Where possible capture an +owner-coherent baseline labelled unknown origin. Ring/map loss marks detail incomplete +and disables exact leak/lifetime conclusions while authoritative summaries continue. +Rebaseline only at documented coherent boundaries; never invent a missing free or claim +all allocations have stacks. Bound cross-thread correlation/maps as well as event rings. + +Comparisons select equal reference-world/drain states and distinguish active/candidate/ +retiring/cache overlap. Use exact ledgers before RSS trends: retained caches are not +automatically leaks, and an unexplained live pin cannot be dismissed as driver caching. + +## 9. Capture, history and export + +### 9.1. State machine + +```text + Idle --> Armed (rolling pre-trigger) --> Recording --> Stopping --> Draining + | ^ | + +---------- manual start ----------+ v + Ready --> Exporting + ^ | + +-----------+ +``` + +Admission reserves archive/metadata/command/result capacity. Reject Busy/BudgetExceeded +without disturbing another capture. One local detailed session is active; summaries +continue. Manual, timed and hitch-triggered modes share the lifecycle. Initial trigger: +declared main/render interval threshold, bounded pre/post windows and cooldown. It detects +a spike, not its cause. Category/trigger selection is recorded before arming. + +Capture/configuration epochs identify records. Stop establishes a cutoff; producer +watermarks and collector-owned chunks seal a fully published prefix. Export never +reads producer arrays. Idle/exited/long-running threads have acknowledgement/finite +timeout rules; do not wait indefinitely for another event. Scopes crossing start/stop +are partial/outside, not silently complete. Control epochs cannot reuse storage until +prior registered producers/consumers no longer reference it. + +GPU results can arrive after CPU stop. Draining retains admitted query/context ownership +until results or a finite declared deadline marks them missing. A capture timeout does +not authorize early resource reuse: retain a bounded renderer retirement owner when +needed and reject new capture admission if it would overlap exhausted slots. + +Manual archive exhaustion stops detail with truncation status; no blocking, silent +prefix overwrite or capacity growth. Armed mode intentionally evicts oldest pre-trigger +history with counts/range metadata. Proposed pre-trigger cap: 8 MiB within the same +32 MiB archive, leaving at least 24 MiB for post-trigger records. Retained duration +depends on event rate; no guaranteed seconds from a byte cap. + +Export failure is separate from capture failure. Ready data survives disk-full, +permissions/cancellation so another destination can be retried. Explicit discard/new +session releases it; capture admission failure preserves previous data. Crash-safe +recovery is outside scope, not justification for synchronous hot-path writes. + +### 9.2. Artifact and comparison + +Version a local bundle containing manifest, bounded chunks, name/clock dictionaries, +capabilities, losses/coverage and summaries. Specify endian/field sizes/count limits, +checksums and supported versions. Compression is bounded export-worker work; offer +uncompressed export. Tracy files remain Tracy-managed, associated through session/build/ +mode metadata where supported, not falsely advertised as the same binary schema. + +Provide CSV summaries/legacy automation plus a tested bounded interchange export for +supported CPU spans/flows. Choose/validate its schema before claiming viewer support. +Uncalibrated GPU domains must not become misleading common-clock trace timestamps. + +Imported files are untrusted: reject invalid bounds/versions/checksums, traversal and +decompression bombs before unsafe allocation. No executed commands, arbitrary URL fetches +or automatic symbol downloads. Export uses a permitted user-selected path and temporary/ +atomic publication where supported; partial output cannot masquerade as valid capture. + +Compare matched workload/options, mode/categories, hardware/driver, viewport, worker +counts, scene/shader identities and cache/warm-up state, or display mismatch warnings. +Retain raw samples/outliers and state populations/exclusions/confidence. Mixed sampling, +loss or truncation cannot produce an apparently exact regression conclusion. + +## 10. Bounds, perturbation and failure behavior + +### 10.1. Proposed development profile + +The following are **initial review caps**, not measurements or engine RAM guarantees. +Calibrate once against finite workloads before freezing qualification. Thereafter, +exceedance fails the profile rather than silently raising limits. New caps do not alter +M1's historical capture capacity or its acceptance evidence. + +| Dedicated local CPU storage | Proposed ceiling | +|---|---:| +| Rings: 32 threads × 4,096 records × 64 bytes | 8 MiB | +| UI snapshots, 512-frame summaries, bounded detail cache | 8 MiB | +| One archive including pre-trigger history | 32 MiB | +| Dictionaries, allocation-ID maps, thread/scope metadata | 4 MiB | +| Export/compression/import scratch | 8 MiB | +| Collector/control/alignment reserve | 4 MiB | +| **Total dedicated local CPU ceiling** | **64 MiB** | + +Validate actual whole layouts/allocator overhead against those terms. A 64-byte record +does not require extra per-record padding. Optional buffers are admitted at safe control +boundaries; Summary must not permanently commit the Detailed ceiling. No hidden large +worker stacks or borrowed/rewound scene arenas. + +Sites/names, thread slots, scope depth, allocation maps, snapshots, exports and GPU pairs +have finite subcaps. Stage P0 publishes concrete entry counts/byte arithmetic before +implementation. More than 32 registered instrumented lanes is partial coverage or another +explicit profile, never a reason to stop engine workers. + +Initial metadata subcaps proposed for P0's layout audit: 4,096 sites/names, 128 categories, +32 registered thread lanes, scope depth 128 per lane, and 16,384 detailed live-allocation +identities. Charge bounded name text, map load-factor/slots, scope stacks and bookkeeping +within the same 4 MiB metadata ceiling. Unsupported cardinality marks detail incomplete; +this is not a 16,384-allocation limit on the engine. Permit one pending export and three +UI snapshot slots; include their actual bytes in the table rather than allocating a +second uncharged archive for each viewer. A new capture cannot replace a Ready/Exporting +archive without explicit discard or completed export/release. + +Summary starts with a separate proposed 16 MiB local ceiling, using only the summary/ +metadata/control terms above and admitting detail rings/archive/export scratch only +when requested. Record core main/render phase totals once per phase/frame; do not +route every fine-grained zone into a supposedly lightweight Summary mode. Proposed +presentation/sampling cadence: charts and UI snapshots 10 Hz, CPU owner statistics 2 Hz, +driver budget/RSS samples 1 Hz, unless an explicit measured profile requests more. +Frame summaries retain original per-frame observations independently of UI cadence. +Sample each owner through its synchronization/owner-thread path; lowering cadence is +not permission to let a background sampler race allocator or renderer state. + +The 64 MiB ceiling is a bounded design starting point for one short detailed capture, +not a claim every developer needs that RAM continuously. A 32 MiB archive holds at most +524,288 fixed 64-byte records before linked metadata overhead; event rate determines +its duration. Admission must report the predicted tradeoff, not promise a fixed-length +capture independent of workload. Smaller/larger profiles require explicit reviewed caps. + +GPU pools are separate: record actual frame-context count, per-queue timed pairs, Tracy +contexts and driver-visible byte estimates when available. Preserve the existing finite +`TimestampPassCapacity` initially. GPU queries and Tracy client/server/network buffers +are measured separately; do not silently include them in the 64 MiB local CPU claim. + +### 10.2. Proposed perturbation gates + +Use optimized builds with matched options/categories/workload, repeated paired batches, +uncertainty, background-load records and no-profiler controls. Freeze event rates before +comparison; do not report clock-scale deltas below noise as precise wins. + +| Mode | Initial review target on the declared profile | |---|---| -| `ZENGINE_PROFILING && ZENGINE_TRACY` | Tracy zones, plots, and allocations | -| `ZENGINE_PROFILING` | `ScopeGuard`/`ProfilerBuffer` fallback and `MemoryProfiler` alloc hooks | -| neither | profiling macros expand to nothing | +| Compiled out | No profiling calls/events, client dependency or profiler-owned runtime storage; verify binary/link artifacts | +| Summary, hidden panel | ≤1% median main/render work increase and ≤0.2 ms extra p99 frame interval | +| Summary, visible panel | ≤0.5 ms p95 panel CPU build; bounded rows/history; graphics cost reported separately | +| Detailed local, no stacks | ≤5% median critical-path work increase and ≤1 ms extra p99 frame interval | +| Tracy / stack / dual-query detail | Independently measured profiles; no inherited Summary overhead claim | + +Relative and absolute limits both apply where listed. These are proposed targets, not +results. An explicitly selected more intrusive session is labelled perturbing and +cannot uncritically certify uninstrumented performance. Existing scene 4/2/8 ms gates +are independent; profiler overhead does not authorize increasing engine limits. + +Local emission performs no file/network I/O, string formatting, allocation, symbolization +or blocking global lock. Measure Tracy's documented behavior separately, not as assumed +allocation-free. Budget collector/export scheduling without taking unbounded existing +resource-worker capacity. When behind, drop detail/defer export instead of stalling owners. + +Expose loss per ring/unregistered thread/scope depth/name table/allocation map/archive/ +GPU pool. Coalesced UI snapshots report age/skips rather than pretending events were +lost. Missing metadata invalidates dependent calculations. No interpolated frees, +completion or display events. Profiler OOM disables optional diagnostics safely, without +changing valid engine allocation outcomes or resource budgets. + +## 11. Builds, Tracy lifecycle, privacy and shutdown + +### 11.1. Proposed mode policy + +| Build | Local facilities | Tracy | Default | +|---|---|---|---| +| Debug developer | Summary/detail available | Optional | Label Debug and validation-layer timing overhead | +| RelWithDebInfo / optimized developer | Available, useful symbols | Optional | Preferred detailed profiling configuration | +| Ordinary Release | Explicit instrumentation profile | Explicit opt-in | Do not assume macros are enabled | +| Shipping | Separately approved diagnostic build only | Not linked/listening by default | No capture/network endpoint without product/security review | + +Reconcile with existing `ZENGINE_PROFILING`/`ZENGINE_TRACY`, not automatic enablement in +every build. Propagate configuration consistently to libraries, editor, inline frontend +and helpers; avoid unrelated public ABI changes. Test OFF/local/Tracy combinations and +reject unsupported configurations during configure rather than surprising linkage. + +Tracy is an optional sink, not the panel's master switch. Pin client/server compatibility. +Document actual discovery/listen address/ports under explicit developer configuration; +do not promise loopback-only without verifying client options. Disconnect/reconnect cannot +cause invalid lifetimes/unbounded retention or recover unknown allocation origins. + +Summary/frame metadata remains available with Tracy. Do not duplicate every CPU event +unless Detailed+Tracy is selected; deep Tracy may disable redundant local zones while +the panel states its reduced coverage. Use documented task/lock APIs; a normal nested +zone cannot begin/end on different threads. + +### 11.2. Privacy and distribution + +Default local exports use module/relative sites and opaque IDs, not raw pointers, +absolute usernames/paths, file contents, secrets or arbitrary user asset text. Opt-in +stacks/symbols and Tracy may reveal more: warn on export and document each sink's actual +redaction capability. The engine cannot claim to sanitize a Tracy file it does not own. + +No automatic uploads. Users select destination/retention/deletion; exported files persist +until deleted while in-memory history is bounded. CI artifacts need declared retention +and sanitized fixtures. Imports/symbols follow untrusted-input rules. Document client +licensing/redistribution separately from distribution of the external Tracy application. + +### 11.3. Shutdown/failure isolation + +Seal capture commands; stop producers/categories via epochs; stop/join workers; drain +published CPU/admitted GPU work only as real device shutdown permits; finalize loss; +cancel/join exporters with finite handling; release readers/contexts before dictionaries/ +storage. Late allocator callbacks must not access dead profiler state. Never join the +collector while holding a lock it needs or wait indefinitely for a lost device. + +Device loss invalidates the GPU epoch. A full disk cannot block rendering; a dead Tracy +connection cannot crash the local panel. Recursion, registration races, partial shutdown +and profiler errors need tests; measurement failure is not successful engine execution. + +## 12. Validation and production acceptance + +Qualification records OS/CPU/GPU/driver, memory, backend, compiler/configuration/symbols, +validation-layer state, mode/caps, fixture/shader IDs and exact commands. Normal CI covers +Linux/Windows/macOS builds and deterministic non-GPU tests. Native rendered/timestamp/ +Tracy evidence is required for each claimed platform capability. + +| Axis | Required cases | Gate | +|---|---|---| +| Transport | Concurrent/wrapping/full rings, slow drain, exit/ID reuse, names/depth | No races/unpublished reads/overwrite; bounded storage and correct loss | +| Frames/clocks | Cross-frame spans, unequal loop rates, invalid clock, long runs | Correct identity/units, explicit N/A; no fabricated aggregation | +| UI | Slow/pinned/closed/reopened views, high DPI/narrow/minimized | Immutable bounded reads, no producer block; visible age | +| Tracy | Local/Tracy/combined, disconnect/reconnect, task flow | Correct configured coverage and context lifetime | +| GPU | Unsupported features/calibration, wrap, delays, failed submit, resize/loss | No extra waits or early reuse; origin IDs and honest availability | +| Memory | Realloc failures, cross-thread free/reuse, arena reset, aliasing, late attach | Correct totals or incomplete status; no double counts | +| Capture | Arm/stop/trigger races, idle thread, full archive, delayed GPU, disk-full | One outcome, bounded drain, sealed valid prefix and retryable export | +| Import | Corrupt sizes/IDs/checksums/version/traversal/expansion | Bounded rejection before unsafe allocation/access | +| Bounds | Exact-cap and cap-plus-one for every pool/queue | No hidden growth or stalled engine work | +| Perturbation | Matched optimized OFF/Summary/Detailed/Tracy | Frozen memory/overhead gates with uncertainty/outliers | +| Lifecycle | Repeated capture/import/export, scene switches, shutdown | Drained baseline restored; no UAF/leak/new GPU validation errors | + +Use deterministic barriers/fault injection, not sleeps as race proof. Run supported +sanitizers and GPU validation, recording unavailable tools and alternatives honestly. +TSan does not detect false sharing; GPU validation is not performance qualification. + +Initial performance suite: steady rendering, replacement, worker-asset pressure and +telemetry-heavy fixtures, including pinned `SampleProject/project.json` and a near-capacity +fixture. Use at least 10 interleaved paired steady batches and 30 first-use/replacement +samples where relevant; predeclare duration/warm-up/uncertainty. These are experimental +minima, not assurance every effect is resolvable. Expand for variance under a recorded +analysis plan, not repeated runs until a desired outcome appears. + +Lifecycle qualification: 100 warm-up + 1,000 bounded capture/stop/export-or-cancel cycles, +restoring the same reference scene and draining owners. With retention disabled, profiler +live ledgers return to warmed baseline; reserved pools stay within caps. RSS/driver usage +are trends. Existing visuals remain semantically unchanged; UI exclusions require +predeclared masks in fixed-camera comparisons. + +Acceptance requires useful non-debugger local controls, honest provenance/degraded modes, +safe optional Tracy, tested portability, bounded storage and perturbation evidence. +APIs or an attractive panel alone are not production acceptance. + +## 13. Delivery plan and exit gates + +These **P0–P5 profiling stages** are unrelated to scene M1/M2/M3 or GitHub priority +labels. Separate PRs/branches must not silently expand #872 or make all profiler work +a prerequisite for cooked rendering/Play. + +| Stage | Deliverable | Exit gate | +|---|---|---| +| P0 — Foundations | Record/ID/clock schema, concrete cap arithmetic, build propagation, safe transport/name ownership | Concurrency/overflow/shutdown tests; legacy capture strategy; baseline profile frozen | +| P1 — Local Performance | Main/render markers, flows, immutable snapshots, bounded CPU/summary UI and shared memory | Useful without Tracy; identity/age/loss visible; Summary/panel gates pass | +| P2 — GPU | Per-pass views, frame/queue epochs, capability status, links, optional calibration | No added waits; unsupported/resize/loss tests and native timing evidence | +| P3 — Memory detail | Attributed owner/resource records, bounded maps, optional stacks, reset/realloc | No double counts/false leaks; coverage/loss/overhead tests | +| P4 — Deep Tracy | Frontend fan-out, task/frame/lock hooks, Vulkan contexts and selected named pools | Connect/disconnect and CPU/GPU lifetime qualification; local summaries retained | +| P5 — Capture product | Live triggers/control, bounded export/import, comparisons, headless/overlay controls | Full failure/security/perturbation/lifecycle evidence and platform handoff | + +Earlier stages require safe capture/state primitives in P0; P5 is product completion, +not a late fix for unsafe exporter/producer access. P3/P4 can proceed after their +foundation/GPU seams stabilize; expose partial capability until their gates pass. +Prefer a small complete first panel over an unbounded custom trace viewer. + +Estimate P0 after the transport/Tracy audit and later PRs from accepted scope/target +profiles. No completion date or assumed access to three platforms is invented here. -`ZENGINE_PROFILING` is currently supplied for Debug and RelWithDebInfo through the engine CMake target. `ZENGINE_TRACY` is an optional root CMake option that defaults to OFF; it is not implicitly enabled for Debug. +## 14. Review decisions and deferred boundaries -`ProfilerBuffer` is a fixed-capacity (4096) static CPU sample/value buffer. It permits concurrent writers and exposes the previous-frame snapshot through `DumpLastFrame` and `DumpLastFrameValues`. Samples are silently dropped at capacity. Its `BeginFrame()` transition and its dump APIs are facilities, not evidence that the main loop currently calls or presents them. +Recommended defaults, not previously approved user decisions: -`MemoryProfiler` records named arenas registered through `MemoryManager::CreateBudgetedArena`, tracks current/peak offsets, emits profile values, and warns above 80% with a 60-second cooldown. It is initialized with the `Bootstrap` owner before tracking. `Engine::MainThreadRun()` calls `MemoryProfiler::Update()` at its main-thread frame boundary, so the editor panel reflects live named-owner current and peak data. The panel publishes separate render-thread snapshots for VMA/driver heap samples, persistent environment resources, and render-graph transients; it intentionally does not collapse these into a CPU-arena total. +1. Keep an integrated Performance panel plus optional Tracy, sharing instrumentation + and immutable telemetry rather than replacing either workflow. +2. Deliver bounded summaries/CPU inspection first, trustworthy GPU correlation and + memory detail next; leave arbitrary-depth analysis to Tracy/platform tools. +3. Review the 64 MiB local Detailed ceiling and section 10 perturbation targets as + calibration starting points; Summary stays smaller and shipping instrumentation OFF. +4. Accept explicit uncalibrated-GPU/presentation-unavailable degraded modes; never + weaken semantics to draw apparently aligned lanes. +5. Default to sanitized local artifacts, without player analytics or remote control; + richer paths/stacks/network diagnostics require informed opt-in. -`RenderGraph` owns per-frame Vulkan timestamp query pools when supported by the selected queue. It records `RGPassTiming` values and exposes `GetLatestPassTimings()` / `IsTimestampProfilingEnabled()`. Query-pool creation failure disables the feature with a warning; the graph timing path is not a UI/profiling overlay by itself. +Bring measured budget/platform/privilege tradeoffs and any shipping/network expansion +back for review. General heap interception, full driver residency, automatic scheduling/ +false-sharing causality, a separate profiler process and fleet analytics remain future +proposals, not implicit completion requirements. -## Production completion criteria +## 15. References -1. Define one non-blocking frame boundary that advances CPU profiling and memory sampling on the correct owning thread. -2. Publish a copied telemetry snapshot to ZUI/debug tooling; never let UI read a live mutable profiler buffer concurrently with its writers. -3. Surface dropped CPU samples, query-pool capacity exhaustion, unavailable queue timestamps, and arena registrations as explicit telemetry. -4. Verify CPU and GPU timing under all supported backends and device-lost, resize, and no-timestamp-feature cases. Timing must never add a GPU wait to normal presentation. -5. Establish build/release policy for profiling symbols, Tracy enablement, capture privacy, and retention. Do not claim “zero overhead” for code paths that still compile instrumentation under `ZENGINE_PROFILING`. +- [Memory budget reference](../memory-budget.md) and [future memory policy](memory-budget.md). +- [RenderGraph GPU timestamp design](../completed/render-graph-redesign.md) and + [renderer resource management](render-resource-manager.md). +- [Runtime reconstruction acceptance](scene-runtime-reconstruction.md), §§9.1–9.3; + [Play ownership](editor-play-mode.md). +- [UI system / earlier DebugOverlay sketch](../completed/ui-system.md): this proposal + supersedes its minimal profiler roadmap, not the implementation history. +- Source: `Profiling/{Profiling,ProfilerBuffer,MemoryProfiler,ThreadCpuTime}`, `RenderGraph`, + `GraphicRenderer`, `MemoryProfilerPanel`, `TLSFSlab`, `GpuAllocator`, `RenderResourceManager` + and `Tetragrama/tests/SceneLifecycleProcess`. +- [Tracy upstream](https://github.com/wolfpld/tracy); use documentation matching the pinned + client instead of assumptions from another version. diff --git a/docs/design/future-plan/scene-cooker-contracts.md b/docs/design/future-plan/scene-cooker-contracts.md index 4e9dcb3f0..a8b4b5b50 100644 --- a/docs/design/future-plan/scene-cooker-contracts.md +++ b/docs/design/future-plan/scene-cooker-contracts.md @@ -13,7 +13,7 @@ format, asset migration, CLI, or game packager is introduced by this design. ## 1. Manifest v1 -The machine-readable shape is [manifest-v1.schema.json](scene-cooker-contracts/manifest-v1.schema.json). +The machine-readable shape is [manifest-v1.schema.json](scene-cooker-manifest-v1.schema.json). Every listed property is required, unknown properties are rejected at every level, and `null` is never an alternative to omission or an empty collection. JSON Schema checks shape; the semantic and byte-level rules below also apply. @@ -1740,11 +1740,11 @@ establish three-host or shipping readiness. ### 9.1. Reviewed wire and package fixtures -[manifest-v1.golden.json](scene-cooker-contracts/manifest-v1.golden.json) fixes +[manifest-v1.golden.json](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/ZEngine/tests/Cook/Fixtures/manifest-v1.golden.json) fixes canonical wire shape, key ordering, the real settings digest, and the final LF. Its synthetic scene digests have no corresponding binary scene or source tree; it is not evidence of a valid package. The -[negative cases](scene-cooker-contracts/manifest-v1.negative-cases.json) specify +[negative cases](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/ZEngine/tests/Cook/Fixtures/manifest-v1.negative-cases.json) specify raw inputs, mutations, and expected outcomes. Keep shape validation separate from graph, fingerprint, byte-integrity, and codec validation. diff --git a/docs/design/future-plan/scene-cooker-contracts/manifest-v1.schema.json b/docs/design/future-plan/scene-cooker-manifest-v1.schema.json similarity index 100% rename from docs/design/future-plan/scene-cooker-contracts/manifest-v1.schema.json rename to docs/design/future-plan/scene-cooker-manifest-v1.schema.json diff --git a/docs/design/future-plan/scene-cooker.md b/docs/design/future-plan/scene-cooker.md index 726d50a22..e005446f1 100644 --- a/docs/design/future-plan/scene-cooker.md +++ b/docs/design/future-plan/scene-cooker.md @@ -386,7 +386,7 @@ collision by first/last entry wins. These failures preserve the prior package. ``` The manifest uses the strict versioned -[v1 JSON Schema](scene-cooker-contracts/manifest-v1.schema.json), with exact +[v1 JSON Schema](scene-cooker-manifest-v1.schema.json), with exact field names, ordering, compatibility, and semantic rules in the [wire contract](scene-cooker-contracts.md#1-manifest-v1). Its records contain: diff --git a/docs/design/future-plan/scene-runtime-m2-profile-candidate.md b/docs/design/future-plan/scene-runtime-m2-profile-candidate.md new file mode 100644 index 000000000..7663ac93b --- /dev/null +++ b/docs/design/future-plan/scene-runtime-m2-profile-candidate.md @@ -0,0 +1,378 @@ +# M2 macOS Vulkan qualification profile candidate + +Status: **draft; not frozen or qualified**. This records a bounded starting point +for the acceptance profile in `scene-runtime-reconstruction.md` sections 9.1–9.5. +The renderer remains RendererOnly. The evidence below is diagnostic and does not +establish the final limits or permit a platform qualification claim. +The user moved strict performance and calibrated-resource qualification to +[follow-up #880](https://github.com/JeanPhilippeKernel/RendererEngine/issues/880) +on 2026-11-17; this does not change the limits or resolve the recorded outliers. + +## Reference environment and fixture + +The measured host was macOS 26.5.2 on an Apple M4 Pro with 48 GiB RAM, using +MoltenVK on the same GPU. The logical viewport was 1280 × 720 and the framebuffer +was 2560 × 1440. The operating-system filesystem cache was uncontrolled. The +fixture was a copy of SampleProject level01 with an added camera and a second +scene UUID sharing the same 32 cooked assets. The capture used validation and +source-access auditing. It is not a cold-disk measurement and does not prove a +near-limit workload. + +The measurement identifies the source scene by SHA-256 +`da6d6c6050200a0d31b17e05ef8b53950ef0ddc705ae4fe7f4e4dfacece9cd31`, +the cooked manifest by +`3d03885391f97d37151e890ee7de525251474e58420413c8cf22d0376afe9b22`, +the runtime executable by +`fc764de14e7e5d6bf8adc96f404942173a04fc82c547c4746d90fda6ec1b9076`, +and the 44-file shader cache by +`dfd8d94eb2a7e0afc9e34f4644052fb2d86de3e7396cb0ff5496d89717d21916`. +The executable predates the later queue-submit diagnostics; its measurements must +not be presented as measurements of that later binary. The [capture archive](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/performance/sampleproject-30-20261117.tar.gz) +contains the report, per-process logs, individual observations and raw stage trace. +It does not include the executable or cooked package. + +## Initial measurements and proposed limits + +Thirty fresh processes and thirty same-process replacements reached an observed +render completion. The fresh p50/p95/maximum were 6.512/6.646/6.662 seconds; +replacement p50/p95/maximum were 7.071/7.428/7.464 seconds. A **12 second** +candidate limit for each gives at least 60% headroom over the observed maximum. +This is a proposal for the stated host and fixture, not a cold-cache or universal +load-time promise. First-use and later replacement results remain separate in the +full report. + +In 275 CPU-boundary and 275 GPU-boundary cancellations of the synthetic stress +fixture, the maximum request-to-cleanup times were 9.799 ms and 28.498 ms. A +**50 ms** candidate cancellation limit gives at least 75% headroom over those +observations. That fixture is smaller than the required near-limit scene, so the +limit is not qualified until cancellation under the declared maximum workload is +measured. These cancellation results used a different executable from the +30-load capture and must be tied to the final build before acceptance. The +[recorded observations](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/lifecycle/lifecycle-cancellation-1100-20261117.json.gz) +match the original report, and the subsequently located [runtime logs](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/lifecycle/lifecycle-cancellation-1100-logs-20261117.tar.gz) +confirm the 1,100-cycle sequence, all 550 cancellation timings, Vulkan validation, +and zero source-access attempts. The runtime SHA-256 was +`d358e3fedeb1784c6bc700e21b9148dd91dc705580faf136c8c89db60ce9ed1e`; +this remains evidence for that older build rather than the current runtime. + +The [current-runtime 1,100-cycle soak](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/lifecycle/lifecycle-final-1100-20261117.tar.gz) +completed 550 cancellations and 550 committed switches with Vulkan validation, +zero source-access attempts, and warmed reference allocation checks passing. +The runtime SHA-256 was +`5187b4d7bf09b3d6d1150856e523c14ef47c4b0dc4bdd159b70f08bc49ce023f`. +Across 275 cancellations at each boundary, CPU cleanup p95/maximum was +3.572/13.440 ms and GPU cleanup p95/maximum was 11.374/124.673 ms. The GPU +maximum exceeds the previously proposed 50 ms limit and must be investigated; +the limit has not been raised or qualified. This uses the small synthetic fixture, +not the declared maximum workload. +Profiling builds now log cancellation request, first render-owner observation, +and terminal mailbox timestamps. An [eight-cycle smoke](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/lifecycle/cancellation-mailbox-diagnostic-20261117.tar.gz) +verified the markers and source/validation checks; it did not reproduce the +124.673 ms outlier. The instrumentation changes the executable, so the preceding +1,100-cycle result remains evidence for its recorded binary, not this later one. +A [100-action diagnostic sample](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/lifecycle/cancellation-handoff-100-20261117.tar.gz) +on the instrumented runtime collected all 25 GPU handoffs with validation and +source auditing. GPU cancellation p95/maximum was 10.159/23.422 ms; the largest +request-to-render-observation interval was 23.259 ms and the largest observed-to- +mailbox-finished interval was 0.044 ms. These independent maxima do not establish +the cause of the earlier 124.673 ms outlier or qualify the proposed limit. +The render loop now pumps only cancelled cooked mailbox work after an invalid +frame acquisition, so minimized or unavailable surfaces need not prevent that +cleanup. The [focused Vulkan test and lifecycle smoke](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/lifecycle/cancellation-no-frame-20261117.tar.gz) +verify cancellation-only pumping does not start new uploads and can drain a +cancelled ticket. They do not reproduce the visible-window outlier or prove the +50 ms limit, and the change creates a later runtime binary. +The focused mailbox test additionally holds an accepted mesh upload behind an +unsignalled GPU timeline, verifies that no-frame cancellation remains pending, +then releases the timeline and drains the ticket without beginning another +frame. This verifies ownership across delayed completion, not a latency bound. +The render loop subsequently moved its cancellation-only mailbox visit ahead of +frame acquisition, without allowing more than one mailbox operation per frame. +An [eight-cycle smoke](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/lifecycle/cancellation-preacquire-20261117.tar.gz) +passed with validation and source auditing; its two GPU cancellation cleanup +observations were 6.698 and 9.990 ms. This does not establish a maximum or prove +that earlier presentation-related delays are eliminated. The runtime SHA-256 +was `f7af4d1d0af102d3d2d2f95dfe3a84c578c431ed34e807662c8946bc7f1475e5`. +The [100-action follow-up](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/lifecycle/cancellation-preacquire-100-20261117.tar.gz) +observed 25 GPU cancellations with p95/maximum cleanup of 10.060/10.212 ms; +the maximum request-to-render-observation was 7.504 ms. Validation and source +auditing passed. This shorter sample is not proof of a 50 ms worst-case bound, +and it does not explain the earlier 124.673 ms observation. + +A targeted 8,192-draw shared-mesh fixture subsequently completed four lifecycle +cycles with one cancellation at each boundary: 0.057 ms for CPU and 13.167 ms +for GPU. Source auditing and Vulkan validation were enabled. This is one sample +per boundary for a synthetic fixture, not a latency distribution or evidence for +maximum asset complexity. The [report](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/capacity/capacity-cancellation-8192-20261117.json) +and [runtime logs](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/capacity/capacity-cancellation-8192-20261117.tar.gz) +are preserved with this candidate. + +The same cooker and runtime binaries accepted the synthetic 8,192-draw scene +and rejected 8,193 draws both at startup and during replacement. After the +replacement refusal, the original scene rendered and a retry succeeded. The +[refusal reports and logs](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/capacity/draw-cap-refusal-20261117.tar.gz) +record source auditing and Vulkan validation. These fixtures share a triangle, +mesh and material; they do not establish limits for asset count, texture size, +material variety, or overall scene complexity. This is a draw-cap observation, +not yet a frozen multi-dimensional profile. + +A further [diagnostic run](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/capacity/material-variety-8192-20261117.tar.gz) +rendered 8,192 draws using 64 distinct cooked material objects and completed four +lifecycle cycles. Its CPU and GPU cancellation observations were 6.772 ms and +6.591 ms respectively. The materials share a texture and most properties, and +the run provides one cancellation sample per boundary. The original run verified +distinct object digests but did not retain its cooked manifest; the probe now +retains and validates the manifest for future captures. This evidence does not +establish a texture-complexity or maximum-asset-count limit. + +A subsequent [texture-overlap probe](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/capacity/texture-overlap-capacity-20261117.tar.gz) +used 8,192 draws, 64 materials and 48 distinct one-pixel textures through four +lifecycle cycles, with auditing and validation enabled. On the same device a +64-texture replacement hit the runtime's 125 live texture slots while the prior +scene and engine resources were still resident. The active generation was +retained, but the stress process then reported failure; this is not a successful +retry test. The probe exposed a separate closure-lease exhaustion bug, now fixed +by releasing decoded texture leases after upload-owned staging accepts the data. +The slot count is shared with engine resources and concurrent candidates, so it +must not be interpreted as a per-scene texture allowance or a guarantee for +larger textures. The 48-texture result is one synthetic observation and does not +qualify the refused 64-texture overlap. + +The [post-delegate-change rerun](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/capacity/capacity-metal-delegate-20261117.tar.gz) +retains the fixture manifest and logs for 8,192 draws, 64 materials and 48 +one-pixel textures over four lifecycle cycles. It enabled Vulkan validation and +source auditing, retained the active rendered scene after both cancellations, +and observed CPU and GPU cancellation cleanup times of 0.049 ms and 32.102 ms. +The two replacement admission-to-submission observations were 13.042 and +12.988 seconds. These are single-run observations for this synthetic workload, +not the 30-sample SampleProject profile or a qualified maximum-workload limit. +The runtime SHA-256 was +`5187b4d7bf09b3d6d1150856e523c14ef47c4b0dc4bdd159b70f08bc49ce023f`. +A [later run after invalid-frame cancellation changes](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/capacity/capacity-current-20261117.tar.gz) +repeated the same 8,192-draw, 64-material, 48-texture fixture and four lifecycle +cycles with validation and source auditing enabled. It observed CPU/GPU cleanup +times of 0.052/13.095 ms and a tracked admitted-GPU peak of 1,532,985,216 bytes. +The retained fixture report records 8,193 entities including its camera; its +two-scene package manifest contains 114 assets. The entities share a synthetic +mesh, so these counts do not establish a general entity or asset limit. +These single observations do not qualify maximum workload latency or full driver +residency; the runtime SHA-256 was +`9b6f46f5aa80920aa0927ef3e79f2393e0725c82f6dd974a69073a82032c8291`. +On the earlier post-delegate runtime (`5187b4d7…`), an [8,193-draw replacement refusal](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/capacity/draw-refusal-metal-delegate-20261117.tar.gz) +preserved the active scene, rendered it after failure, and accepted a valid +retry. Vulkan validation and source auditing were enabled. This verifies the +draw limit's recovery path for the synthetic fixture, not every resource cap. +The [current-runtime rerun](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/capacity/draw-refusal-current-20261117.tar.gz) +also refused 8,193 draws, retained and rendered the active scene, and successfully +retried, with validation and source auditing enabled. The runtime SHA-256 was +`9b6f46f5aa80920aa0927ef3e79f2393e0725c82f6dd974a69073a82032c8291`. + +An [editor-hosted package-failure smoke](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/failures/package-failures-current-20261117.txt) +on the same runtime rejected missing, corrupt, wrong-target and wrong-format +packages, as well as an incorrect expected draw count. The fixture's authoring +copy was removed before launch and Vulkan validation was required for the +successful initial run. This runner does not preserve per-process logs for these +failure cases or apply the standalone source-access audit; it supplements rather +than replaces the detailed failure and standalone evidence. +The [standalone failure run](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/failures/standalone-package-failures-20261117.tar.gz) +then exercised those cases on the same runtime with the authoring copy removed, +source auditing and Vulkan validation enabled. Each damaged package failed +before publication, shut down cleanly and reported zero denied source, registry +and import accesses. The archive retains the individual process logs; the opt-in +`CookedDesktop.StandalonePackageFailures` test repeats this coverage. + +A separate [refusal-and-retry probe](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/capacity/texture-refusal-retry-20261117.tar.gz) +started with 48 distinct textures, attempted a 96-texture replacement, and hit +the same 125-slot limit. It observed the old scene render after the failure and +successfully retried the 48-texture scene, with source auditing and Vulkan +validation enabled. This establishes recoverability for that fixture, not +support for a 96-texture replacement or a universal texture count. +The [typed-outcome rerun](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/capacity/texture-typed-refusal-20261117.tar.gz) +also checks that the retained failure reports the preparation stage and the +`Resources` error before acknowledgement and retry. +The [post-delegate-change rerun](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/capacity/texture-refusal-metal-delegate-20261117.tar.gz) +repeated the 48-active/96-candidate texture overlap refusal and valid retry on +runtime SHA-256 `5187b4d7bf09b3d6d1150856e523c14ef47c4b0dc4bdd159b70f08bc49ce023f`, +with validation and source auditing enabled. This remains a recovery check, +not a qualified per-scene texture capacity. + +The two cooked scenes currently prepare separate texture handles even when their +asset UUIDs and object digests match. Scene retirement queues each owned handle +for release and seals it against further writes. Any future reuse must therefore +match both identity and digest, retain an explicit shared owner across active, +candidate and GPU consumers, and defer release until the final owner retires. +Reusing a handle while retaining the current per-scene release behavior would be +unsafe. Until such sharing exists, profile admission must account for both sets +of texture slots during replacement. + +The [texture-lease regression evidence](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/capacity/texture-lease-regression-20261117.tar.gz) +contains the original failed 64-texture startup log, the targeted upload-ownership +test, two Vulkan cancellation/refusal recovery tests, and focused Linux sanitizer +results. The original startup failure was generic and did not itself identify the +exhausted lease limit; the ownership test and subsequent successful initial +64-texture load support the fix. The Linux container compiled the affected files +from the current tree in an older cached source snapshot and had no GPU; it is not +a full current-worktree or rendered Linux qualification. + +SampleProject replacement peaked at 173,265,956 bytes in the tracked CPU pool, +1,894,241,664 admitted GPU bytes, and 389 of 2,047 retirement entries. GPU domain +peaks are recorded individually in the report and need not have occurred at the +same time. GPU admitted bytes are an allocator ledger, not driver residency or +whole-process memory. The runtime's existing safety ceilings include a 1 GiB CPU +pool, a 256 MiB preparation-staging pool, a 16 MiB metadata pool, and a 4 GiB +total GPU admission limit; these are implementation ceilings, not calibrated +supported-scene-size claims. +The standalone runtime also configures `MaxAssets` to 8,192 and both +`MaxObjectBytes` and `MaxMeshWorkspaceBytes` to 512 MiB. These are separate +input/workspace ceilings, not additional guarantees that an 8,192-asset scene or +512 MiB object fits in the shared CPU pool. Their combined maximum-workload and +limit-plus-one behavior is not established by the synthetic texture fixture. + +The earlier capture's per-domain GPU figures are not suitable for calibrating +texture and render-target limits: cooked 2D images were classified as render +targets because they had attachment capability. The renderer now explicitly +charges cooked textures and environment maps to the texture domain, and a +focused budget-refusal test and source-free smoke capture verified that path. +Recollect domain high-water evidence on the final build before freezing limits. +A [single corrected-domain capture](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/performance/sampleproject-corrected-domains-20261117.tar.gz) +on runtime SHA-256 `14af15626c422b8d299fc2483c7a650067fc65028d510565289f0122a6533b5e` +reported replacement peaks of 316,097,536 texture bytes and 131,975,552 +render-target bytes. The domain peaks are independent maxima, not a simultaneous +sum, and exclude driver residency. Its matched trace found 73 replacement frame +intervals over the steady p95 plus 8 ms, including a roughly one-second queue +submission. This is a failing diagnostic observation, not a qualified result or +proof of why those intervals occurred. One capture does not replace the required +30-sample evidence on the final build. + +A [post-replacement steady trace](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/performance/sampleproject-post-steady-20261117.tar.gz) +captured 120 submitted frames on each side of the replacement. Render-frame p95 +was 25.451 ms before, 25.825 ms during and 25.628 ms after replacement. The +similar elevated steady intervals are evidence that a slow presentation state +can persist without a load in progress; they do not explain the separate +one-second initial submission wait or prove other captures' outliers external. +This is one diagnostic run and does not qualify the performance profile. + +## Drawable-wait investigation + +On the later diagnostic build, an initial-load interval of 1,008.491 ms contained +a 1,006.153 ms Vulkan queue submission, of which 1.202 ms was calling-thread CPU +time. The [trace report](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/performance/submit-diagnostic-20261117.json) +records the full diagnostic. A [separate fresh-load thread sample](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/performance/macos-drawable-stack-20261117.txt.gz) +showed the render thread inside MoltenVK's `MVKPresentableSwapchainImage` waiting +in `CAMetalLayer nextDrawable`. This stack is from another process, not a stack +captured during the exact traced interval; it does not by itself establish the +cause of that interval or exclude loader influence. + +A second [trace](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/performance/submit-async-probe-20261117.json.gz) +used `MVK_CONFIG_SYNCHRONOUS_QUEUE_SUBMITS=0` with the same executable, shader +cache and viewport. It did not show the one-second submission interval, but it +reported 90 replacement intervals over its matched steady threshold, primarily +overlapping image acquisition. This configuration is an experiment, not a +recommended fix or a qualifying comparison. Both traces are single captures; +presentation scheduling and window visibility were not controlled. + +A later [window-visibility trace](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/performance/sampleproject-window-visibility-20261117.tar.gz) +recorded the macOS window as fully occluded through its steady and replacement +windows. A separate [recovered trace](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/performance/sampleproject-occlusion-transition-20261117.tar.gz) +showed all 60 replacement outliers after the window became occluded, with none +before. Its collector stopped after capture because it did not yet recognize the +visibility metric, so the recovered report lacks a verified manifest identity. +These observations identify a concrete +confounder, not proof that every stall is unrelated to loading. The timing tool +can now require visibility throughout all measured windows; a hidden or unknown +window is refused rather than treated as visible-window qualification. A valid +visible-window run remains necessary. Use `--trace-steady-frames 120 +--require-visible-trace` with `tests/scene/sampleproject_cooked_timing.py` to check +the initial, pre-replacement, replacement and post-replacement windows. The +strict check requires macOS occlusion observations for all four trace windows +and for every timed load through observed render completion. It preserves the +diagnostic trace when trace-window visibility is not proven. Each test process +waits up to 15 seconds for its window to become visible before admitting the +scene; it does not force the window in front of other applications. Earlier +timing reports only checked the separate trace process and cannot establish +visibility for all of their individual timing samples. + +In the visibility trace's initial load, the roughly 1,009 ms frame interval began +while the window was visible, and the window became fully occluded about 12 ms +into that interval. The long submission and visibility transition therefore +overlap closely. This supports window occlusion as a possible explanation; it +does not prove the cause or substitute for a visible-window measurement. + +A later [visible-window trace](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/performance/sampleproject-visible-user-20261117.tar.gz) +passed the visibility check in all four timing windows on runtime SHA-256 +`b8e3af59b4a1e7059b24f1248314ea204686de0f8b340f7a2b37eb7d37f8d211`. +Replacement had no intervals over steady p95 plus 8 ms, while initial load had +two, at 41.817 ms and 16.758 ms against a 16.482 ms threshold. The larger interval +overlapped a 40.112 ms Vulkan queue submission. This rules out full-window +occlusion as an explanation for those two observations, but does not establish +their cause. The single trace is not the required 30-sample qualification. + +The [post-delegate visible-window run](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/performance/sampleproject-visible-final-20261117.tar.gz) +completed 30 fresh loads and 30 replacements with visibility verified during +each measured load. Fresh observed GPU-completion p50/p95/maximum were +5.443/5.468/5.472 seconds; replacement values were 5.362/5.413/5.422 seconds. +The filesystem cache was uncontrolled, and completion is a polling observation, +not presentation. In the separate visible trace, initial loading had one +33.394 ms render interval against a steady-p95-plus-8-ms threshold of 16.525 ms; +31.586 ms overlapped queue submission. Replacement had no such outliers in that +trace. The cause remains unproven, so this run does not satisfy the frame-stall +qualification or show that the delegate change fixed the issue. +Inspection of that outlier shows 0.497 ms of measured render-thread CPU time +inside the 31.585 ms queue-submit call; the remaining 31.088 ms is non-CPU time, +which can include blocking or descheduling. The immediately preceding recorded +cooked upload visit took 0.032 ms. These observations narrow the location of the +delay but do not identify its cause or prove whether loading contributed. +The original report marked two tiny environment scopes in each loading window +as unassigned because their complete frame ended just after the window. The +analyzer now includes complete boundary-crossing frames; re-analysis assigns all +four scopes, leaving the initial/replacement environment maxima unchanged at +3.438/0.213 ms. This corrects attribution, not the unexplained frame outlier or +the absence of a current-build qualification trace. + +A temporary macOS experiment requested three swapchain images instead of two, +within surface limits. Its [visible trace](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/performance/sampleproject-triple-drawable-20261117.tar.gz) +confirmed three images and recorded initial/replacement interval maxima of +16.926/18.327 ms, below that run's steady-p95-plus-8-ms threshold of 19.269 ms. +Its steady p95 was 11.269 ms, versus 8.525 ms in the earlier two-image trace. +These are individual runs, not controlled distributions; they neither prove +improvement nor explain the original outlier. The experiment was not retained +as a renderer default, and does not qualify the frame-stall requirement. + +After configuring the Metal layer's hosting view as its delegate (commit +`b6477ba0`), a [fixed-window SampleProject capture](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/visual/sampleproject-metal-delegate-20261117.tar.gz) +confirmed that the delegate was installed in both source and cooked runs. Vulkan +validation was enabled, the cooked source-access audit passed, and source and +cooked captures matched pixel for pixel. Their pixel hash also matches the +existing fixed-viewport reference, which the user subsequently approved. This +run did not measure the visible-window stalls or establish a performance +improvement. +After the invalid-frame cancellation change, a [new fixed-window capture](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/visual/sampleproject-approved-current-20261117.tar.gz) +matched the user-approved reference and the source render pixel for pixel, with +Vulkan validation and standalone source auditing enabled. Its runtime SHA-256 is +`9b6f46f5aa80920aa0927ef3e79f2393e0725c82f6dd974a69073a82032c8291`. +This confirms this fixture's reference match; it does not qualify the outstanding +performance and resource limits. +The [post-scheduling capture](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/visual/sampleproject-approved-preacquire-20261117.tar.gz) +also matched both the source render and approved reference pixel for pixel, +with validation and source auditing enabled. Its runtime SHA-256 is +`f7af4d1d0af102d3d2d2f95dfe3a84c578c431ed34e807662c8946bc7f1475e5`. + +## Work required before freezing + +- Pin the final executable, cooker, shaders, driver and fixture manifests; preserve + the raw evidence. Declare measured entity, draw and asset caps and verify each + cap and cap-plus-one on the final build. +- Establish finite scene/document/scratch/cache, staging, GPU residency and + queue limits including active/candidate overlap and pinned old versions. Record + headroom and account for memory outside the allocator ledger. +- Measure load and cancellation latency at the declared workload limits. Verify + 4 ms aggregate loading work, 2 ms guarded commit, and the 8 ms matched-steady + frame-stall requirement without discarding unexplained outliers. +- Resolve the initial-load Vulkan submission outlier. A separate diagnostic + sample observed MoltenVK waiting in `CAMetalLayer nextDrawable`; that is not + yet proof that every observed outlier is unrelated to loading. An asynchronous + MoltenVK submission probe shifted waiting to image acquisition and produced + replacement outliers, so it is not an accepted workaround. +- Complete semantic visual coverage against the approved fixed-viewport backend + reference. Pass the applicable Linux, macOS and Windows CI gates; claim rendered + qualification only for platforms with actual rendered evidence. diff --git a/docs/design/future-plan/scene-runtime-reconstruction.md b/docs/design/future-plan/scene-runtime-reconstruction.md index 7e8750c3b..3d3a01c6c 100644 --- a/docs/design/future-plan/scene-runtime-reconstruction.md +++ b/docs/design/future-plan/scene-runtime-reconstruction.md @@ -451,8 +451,9 @@ process RSS, and GPU allocations; report those measurements separately. GPU backing allocation admission uses a single fixed 16,384-record ledger for buffers and images, including the staging ring and one-shot staging. Each native -object first reserves bookkeeping, then charges its queried Vulkan memory -requirements before VMA allocates backing memory. Pending reservations count +object first reserves bookkeeping, then conservatively charges its aligned Vulkan +memory requirements before VMA allocates backing memory. An allocation whose +reported size exceeds that reservation is discarded before publication. Pending reservations count against both aggregate and domain ceilings; native allocation/binding/view failure returns the reservation. An alias retains the same backing record rather than charging its memory twice. Retiring objects keep their charge until the final @@ -608,6 +609,15 @@ their implementation/service adapters and qualification remain milestone 3 work. ## 9. Delivery and acceptance gates +The user decided on 2026-11-17 to complete M2's functional scope and track strict +performance and calibrated-resource qualification separately in +[follow-up #880](https://github.com/JeanPhilippeKernel/RendererEngine/issues/880). +The limits and evidence requirements in sections 9.1–9.3 remain the follow-up's +qualification contract; they have not been met or weakened. M2 retains bounded +runtime admission and failure recovery, source-independent rendering, and the +reviewed visual reference. This decision does not claim a qualified production +resource profile or rendered support on unmeasured platforms. + ### 9.1. Supported qualification scope Qualification covers single-active-scene replacement on the existing desktop @@ -668,6 +678,9 @@ Timing/visual thresholds can be revised during proposal review or documented calibration, never retroactively weakened to pass an unexplained regression. No build results or benchmark history accumulate in this contract; retain the profile and evidence as versioned test/PR artifacts. +The initial macOS Vulkan calibration and its open gates are recorded in the +[M2 profile candidate](scene-runtime-m2-profile-candidate.md); it is not frozen +and does not establish qualification. ### 9.3. Acceptance matrix @@ -681,7 +694,7 @@ fixtures, reference reclamation and its applicable timing/visual checks. | Guarantee | Required exercise | Pass condition | |---|---|---| | Reconstruction | Fresh source/cooked process; UUIDs, components, hierarchy, camera and settings | Semantic values agree; runtime handles are rebuilt, not compared as persisted identity | -| Complete rendering | SampleProject opaque/transparent materials, textures, lights/environment | All required bindings ready; semantic checks and frozen visual profile pass | +| Complete rendering | SampleProject opaque/transparent materials, textures, lights/environment | All required bindings ready; semantic checks and reviewed backend reference pass | | Source independence | Deny source/registry/import paths; monitor VFS access | A cooked scene renders with zero source/import lookups | | Recoverable failure | Inject failure at each reservation/read/decode/bind/upload/preflight boundary | No partial publication; old bindings/generation/authored state preserved and next valid request succeeds | | Exactly-once outcome | Deterministic barriers around Ready/commit; cancellation, newer request, shutdown | Exactly one terminal result; no stale publish; TooLate after commit | @@ -692,7 +705,7 @@ fixtures, reference reclamation and its applicable timing/visual checks. | Integrity/capability | Malformed/truncated objects, path escape, changed file, versions/targets, known-but-unsupported services | Stable diagnostic and no source fallback; decode uses verified bytes | | Lifecycle stress | 100 warm-up then 1,000 alternating load/cancel/switch cycles, including shutdown races | Zero stale publications, lost outcomes, use-after-free or renderer validation errors | | Reclamation | Repeat with eligible cache retention disabled; restore the same reference scene and drain uploads/readers | Tracked live allocations/pins return to the warmed reference baseline; reserved pools remain within caps | -| Performance | Timed runs in the frozen profile; trace both CPU critical paths and resource readiness | Per-stage, frame-stall, total-load and high-water limits pass; all outliers accounted for | +| Performance and calibrated resources (follow-up #880) | Timed runs in the frozen profile; trace CPU critical paths and resource readiness | Per-stage, frame-stall, total-load and high-water limits pass; all outliers accounted for | | Device loss | Inject device loss during preparation/retirement | Host enters fatal/device-loss policy; no false preserved-renderability or successful-fence claim | Fault injection uses deterministic barriers and bounded test timeouts, not sleeps @@ -1262,6 +1275,13 @@ Preparation clears that gate only after observing completion; submitting a consumer that waits on the upload cannot prematurely mark its inputs ready. Records do not grow with requests or frames and are cleared before producer semaphore destruction at idle/device-loss shutdown. +After checking the exact ticket, accepted graph acquires retain their consumer +timeline in the bounded texture record. Texture readiness requires observed producer and +graphics-family acquire completion plus the current descriptor and a version +pin. Ticket discard is not an acquire proof. Cancellation and timeline teardown +wait for accepted acquire work to retire. Untyped bindless consumers and passes +using ticketed images run on the graphics family because these uploads release +ownership there; other queue roles require an explicit later handoff. Texture retirement uses the same completion proof even when its acquire ticket has already been acknowledged. Removing that ticket cannot admit destruction while the upload is still in flight; errors retain the gate until successful @@ -1715,7 +1735,7 @@ environment: ```sh SceneLifecycleProcess project.json /Scenes/GizmoActors.zescene /Scenes/level01.zescene evidence samples.csv 1000 100 zero-retention -python3 Scripts/validate_scene_lifecycle.py evidence/samples.csv --cycles 1000 --warmup 100 --zero-retention --stages evidence/samples-stages.csv +python3 tests/scene/validate_scene_lifecycle.py evidence/samples.csv --cycles 1000 --warmup 100 --zero-retention --stages evidence/samples-stages.csv ``` Without `zero-retention` the runner is a mixed-lifecycle soak. With that option it @@ -1833,7 +1853,7 @@ alone does not claim deterministic pixel qualification. Unsupported sanitizers must retain their recorded limitation and alternative coverage. Windows execution remains user-owned. -`Scripts/compare_scene_capture.py` compares bounded P6 RGB8 captures produced by +`tests/scene/compare_scene_capture.py` compares bounded P6 RGB8 captures produced by the Vulkan screenshot layer with `VK_SCREENSHOT_FORMAT=UNORM`. Opaque presentation alpha is implicitly 255. Each explicitly named viewport/object rectangle uses the fixed 4/255 channel and 0.5% pixel thresholds; neither automatic registration, @@ -1864,8 +1884,147 @@ from that failing layer can be stale data, not proof of deterministic rendering. - Use `SampleProject/project.json` for editor regression/visual coverage and a cooked copy for the source-free rendered test. Check transparent materials, lighting/environment, hierarchy transforms, and replacement against references. -- Freeze the qualification profile and pass the applicable acceptance matrix; - report RendererOnly scope rather than implying supported gameplay services. +- Report RendererOnly scope rather than implying supported gameplay services. + Freeze and qualify the performance and calibrated-resource profile in + [follow-up #880](https://github.com/JeanPhilippeKernel/RendererEngine/issues/880). + +The opt-in `ZENGINE_ENABLE_COOKED_DESKTOP_TESTS` CMake option registers the +`CookedDesktop.*` CTest group. It requires a working desktop Vulkan session and +the built `Obelisk` and `SceneCookProcess` targets. Run it with +`ctest --test-dir -C -R '^CookedDesktop\.' --output-on-failure`. +This group exercises source-free startup, package faults, replacements, +supersession while waiting or during CPU/GPU work, and CPU/GPU cancellation, +staging refusal, or injected rejection immediately before a cooked upload +queue submission followed by retry. The latter exercises the rejection cleanup +path but does not simulate device loss or a driver-returned error. Its generated fixture +and local results do not replace the profile, visual references, or supported +platform CI gates above. + +The editor-hosted cooked launch can opt into a full-window authored-camera view with +`--cookedRuntimeView`; `--cookedCamera ` selects a camera explicitly, +otherwise exactly one main camera is required. It prepares and renders using +that camera and presents the scene without editor panels, grid, or gizmos. +`--cookedStandalone --cookedPackage --cookedScene ` launches +the renderer-only application without an editor session; it uses the unique +main camera unless `--cookedCamera ` is supplied. The runtime application +can write a bounded scene-target RGB8 capture with `--cookedCapture ` and +`--cookedExitAfterFrame`; this readback does not prove swapchain presentation. +`tests/scene/sampleproject_cooked_runtime.py` cooks a copied `level01` with a +fixed added camera, captures the source scene with that camera, removes its +authoring copy, and compares a standalone cooked capture at the same viewport. +Its `--force-opaque` negative control changes copied blend materials to opaque; +`--different-from` and `--minimum-different-pixels` can require a visible +difference from a declared baseline in both source and cooked captures. +`--baseline-evidence` additionally checks that the baseline pixels, executable, +shader cache and device identities match the control capture; without it the +report marks baseline identity unverified. A +[local control](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/visual/sampleproject-transparency-control-20261117.tar.gz) +changed 3,988 pixels while source and cooked still matched. That difference was +below the whole-frame tolerance, showing why visual parity alone cannot certify +transparency. This control does not establish the correctness of every blend +material or replace a reviewed reference. +The `--no-directional-light` control disables the copied scene's light. A +[local capture](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/visual/sampleproject-light-control-20261117.tar.gz) +changed 2,737 pixels from the provenance-recorded normal source capture while +source and cooked control images matched exactly. It verifies that the light +affects that capture, not that every lighting component is correct. +Its `--hierarchy` variant parents the SampleProject mesh under an existing actor, +moving the mesh translation to the parent so the expected world transform stays +the same. A [local diagnostic capture](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/visual/sampleproject-hierarchy-20261117.tar.gz) +matched source and cooked images and an earlier flat source capture pixel for +pixel. This is not a reviewed backend reference or final-build qualification. +With `--hierarchy --replacement`, the runtime first renders the original flat +scene, replaces it with the parented version under a distinct scene UUID, and +captures the replacement. A [local diagnostic capture](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/visual/sampleproject-hierarchy-replacement-20261117.tar.gz) +matched the parented source image pixel for pixel with source auditing and Vulkan +validation enabled; it does not replace reviewed visual qualification. +It can also compare the cooked capture with an externally reviewed reference and +predeclared regions using `--reviewed-reference`, `--reference-review`, and +`--region`. Supplying an identifier records a review claim; the script does not +itself establish that the reference was reviewed or that regions cover every +required visual semantic. +The macOS Vulkan SampleProject candidate and review notes are in +`ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-candidate.*`. +The user accepted the character and lighting as a candidate; the review is not +acceptance of every material, transparency, hierarchy, or replacement case. +The separate fixed-viewport candidate is +`ZEngine/tests/Rendering/Fixtures/sampleproject-level01-macos-vulkan-fixed-candidate.*`. +The user approved this fixed-viewport image as the macOS reference. Its original +capture did not record executable or shader identities. A [later capture](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/visual/sampleproject-provenance-20261117.tar.gz) +on a recorded binary and shader cache matched its pixels exactly; the reference +approval does not by itself qualify every semantic or a later build. +A [second capture on that same binary and device](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/visual/sampleproject-provenance-repeat-20261117.tar.gz) +also matched the first pixel for pixel; two captures provide limited noise +evidence, not a general guarantee of deterministic rendering. +A [capture after the cooked GPU-domain accounting correction](https://github.com/JeanPhilippeKernel/RendererEngine/blob/0532bb35d45f2aae42fa2dbb3cd9e3d3711e1165/tests/scene/evidence/visual/sampleproject-domain-visual-20261117.tar.gz) +matched the candidate and its source capture pixel for pixel, with the later +binary and device identities recorded. +An optional `--presence name:minimum-count` requires both source and cooked +captures to contain that many dark pixels within a declared region. This is a +coarse missing-object smoke check; it cannot prove identity, material, or lighting. +Its output is smoke evidence, not a reviewed reference or complete semantic +qualification; the camera modification is recorded explicitly. +`tests/scene/cooked_scene_parity.py --standalone --require-validation` also exercises +generated opaque, transparent (`--blend`), and parent/child (`--hierarchy`) +fixtures with an authored camera and a fixed 1280x720 logical window. It removes +the source tree before standalone launch, requires a zero source-access audit, +and checks engine and console validation output. For hierarchy, `--flat-reference` +can additionally compare an equivalent flat source capture. These are semantic +smoke checks; they do not replace reviewed references or SampleProject coverage. +The blend variant can also use `--opaque-reference` to require an intermediate +colored region over the matching opaque fixture; this is a coarse visual check, +not a proof of the exact compositing equation. +`tests/scene/sampleproject_cooked_timing.py` collects fresh-process and same-process +timings from a copied level01 with an added camera and a second scene UUID. The +two scenes share assets; the report records the fixture and executable hashes, +viewport, cache conditions, validation status, source audit and observed render +completion. It is diagnostic evidence, not a calibrated profile or presentation +measurement. +The standalone `--cookedAuditSource` probe denies and counts engine-context +project VFS read/root-acquisition attempts, registry lookup entry points, and +import enqueues from runtime initialization through joined-worker shutdown. +The separate cooked-package context and engine-asset namespace remain available. +This is not an operating-system-wide file-access monitor and does not cover +engine bootstrap before the audit is enabled. +It also admits a replacement while retaining the active scene; a newer request +supersedes an older preparing or waiting request and starts after cleanup. Its +optional request ID exposes the terminal result until the caller acknowledges +it, including when acknowledgement precedes worker or GPU cleanup. Its +desktop smoke tests exercise repeated +replacements, missing and corrupt scene failures, staging-budget refusal, an +injected upload submission rejection, and cancellation at CPU-worker and recorded-upload boundaries, each +followed by an outgoing scene submission and a successful retry. These tests +observe accepted scene submissions; qualified +window presentation and visual evidence remain outstanding. Runtime waiting, +CPU and recorded-upload supersession are exercised; the broader failure matrix +still needs coverage. A separate probe injects device-lost state after upload +recording and verifies fatal shutdown without replacement publication; it does +not reproduce a driver-originated loss or cover every retirement race. + +The standalone smoke harness can also export per-generation admission-to-first- +accepted-submission and bounded-memory samples, plus an optional stage trace. +`--observe-gpu-completion` additionally polls the timeline for the generation's +first accepted render submission and records when completion was observed. That +timestamp is an upper bound on completion, includes polling delay, and is not +proof of swapchain presentation. Device loss or a missing timeline fails the +observation instead of being reported as a completed render. +`--steady-frames` records an intervening settled window before replacement; +`tests/scene/analyze_cooked_runtime_trace.py` checks and summarizes those windows. +For fresh-versus-replacement diagnostics, `--include-alternate` gives fresh +processes the same two-scene package as the replacement run, and +`tests/scene/analyze_cooked_timing_samples.py` requires matching manifest, binaries, +host, and validation status before summarizing their separate distributions. +These observations are diagnostic until the fixture, limits, cache conditions, +and matched steady-rendering baseline are qualified. + +`tests/scene/cooked_capacity_probe.py` generates an offline-cooked scene with one +shared mesh/material, an authored camera, and a requested number of instances. +It can check the declared 8,192 draw capacity or require clean startup refusal +at 8,193, with validation and source-access auditing enabled. Its +`--replacement-refusal` mode checks that the outgoing scene renders after an +8,193-instance candidate is rejected and that a valid retry succeeds. This +synthetic fixture does not represent complex-asset capacity or a calibrated +memory profile. ### 9.6. Milestone 3 — Play and simulation integration diff --git a/docs/design/index.md b/docs/design/index.md index cfaf82774..c23e63fe0 100644 --- a/docs/design/index.md +++ b/docs/design/index.md @@ -19,6 +19,7 @@ proposed work, or implementation history. - [Scene cooker review proposal](future-plan/scene-cooker.md) - [Scene cooker v1 contracts](future-plan/scene-cooker-contracts.md) - [Renderer resource-management proposal](future-plan/render-resource-manager.md) +- [Profiling: in-editor Performance, CPU/GPU/memory and Tracy proposal](future-plan/profiling.md) - [Zodiac Engine Hub / Panzerfaust integration](future-plan/panzerfaust.md) ## Proposal / RFC documents diff --git a/docs/project/rfcs.md b/docs/project/rfcs.md index fac675137..7785166bc 100644 --- a/docs/project/rfcs.md +++ b/docs/project/rfcs.md @@ -16,6 +16,7 @@ not define shipped behavior. Implemented contracts belong in the - [Editor undo / redo](../design/future-plan/editor-undo-redo.md) - [Editor play mode](../design/future-plan/editor-play-mode.md) - [Render resource manager](../design/future-plan/render-resource-manager.md) +- [Profiling: in-editor Performance, CPU/GPU/memory and Tracy](../design/future-plan/profiling.md) - [Build and distribution integration](../design/future-plan/build-integration.md) ## Recently completed designs diff --git a/tests/scene/README.md b/tests/scene/README.md new file mode 100644 index 000000000..483ffe645 --- /dev/null +++ b/tests/scene/README.md @@ -0,0 +1,23 @@ +# Scene runtime tools + +This directory contains the scene-runtime smoke, parity, capacity and timing +tools, plus the validators shared by their reports. Paths in the CLI examples +below are relative to the repository root. + +- `cooked_scene_smoke.py` exercises standalone and editor-hosted cooked scenes, + replacement, cancellation and injected failures. +- `cooked_scene_parity.py` and `sampleproject_cooked_parity.py` compare source + and cooked scene behavior. `sampleproject_cooked_runtime.py` captures and + compares the rendered SampleProject scene. +- `cooked_capacity_probe.py` creates synthetic capacity fixtures. + `sampleproject_cooked_timing.py` collects timing observations. +- `analyze_cooked_runtime_trace.py` and `analyze_cooked_timing_samples.py` + analyze those observations. `compare_scene_capture.py`, + `validate_scene_lifecycle.py`, `verify_scene_sanitizer_results.py` and + `cooked_capture_identity.py` provide shared validation and provenance checks. + +Run a tool with `python3 tests/scene/.py --help`. Run the Python checks +with `python3 -m unittest discover -s tests/scene/unit -v`. + +Raw M2 observations and their limitations are indexed in +[`scene-runtime-evidence`](evidence/README.md). diff --git a/tests/scene/analyze_cooked_runtime_trace.py b/tests/scene/analyze_cooked_runtime_trace.py new file mode 100644 index 000000000..95a7d5ddb --- /dev/null +++ b/tests/scene/analyze_cooked_runtime_trace.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Summarize standalone steady and load windows without claiming qualification.""" +import argparse +import json +from pathlib import Path +import sys + +from validate_scene_lifecycle import capture_records, distribution, read_stages + +STAGES = ( + "SceneLoad.Main.FrameWorkMs", + "SceneLoad.Main.FrameIntervalMs", + "SceneLoad.Main.RuntimeUpdate", + "SceneLoad.Main.PlatformEvents", + "SceneLoad.Main.OverlayBuild", + "SceneLoad.Main.SceneSynchronization", + "SceneLoad.Main.PrepareScene", + "SceneLoad.Main.PublishFrame", + "SceneLoad.Main.RuntimeGuardedCommit", + "SceneLoad.Render.FrameWorkMs", + "SceneLoad.Render.FrameIntervalMs", + "SceneLoad.Render.CookedUploadVisit", + "SceneLoad.Render.EnvironmentPreparation", + "SceneLoad.Render.QueueSubmit", + "SceneLoad.Render.AcquireImage", + "SceneLoad.Render.PresentWSI", + "SceneLoad.Render.RecordScene", +) + + +def frame_scope_diagnostics(records, window, frame_stage, scopes_to_report): + """Associate fully contained scopes with a measured frame on one thread.""" + # Include complete frames crossing either window boundary. Scopes ending + # inside the window can belong to a frame that ends just after it. Count + # each included scope in full rather than dropping its work. + frames = sorted((timestamp, value) for kind, name, timestamp, value in records + if kind == "value" and name == frame_stage and timestamp >= window[0] + and timestamp - value * 1_000_000 <= window[1]) + names = {name for name, _ in scopes_to_report} + scopes = [(name, timestamp, value) for kind, name, timestamp, value in records + if kind == "scope_ns" and name in names + and window[0] <= timestamp <= window[1]] + result = {"frames": len(frames), "partial_scopes": 0, "unassigned_scopes": []} + for stage, label in scopes_to_report: + per_frame = {} + for name, end, duration in scopes: + if name != stage: + continue + candidates = [index for index, (frame_end, work_ms) in enumerate(frames) + if frame_end - work_ms * 1_000_000 <= end - duration and end <= frame_end] + if len(candidates) == 1: + index = candidates[0] + per_frame[index] = per_frame.get(index, 0) + duration / 1_000_000 + else: + result["partial_scopes"] += 1 + result["unassigned_scopes"].append({"stage": name, "end_ns": end, "duration_ms": duration / 1_000_000}) + values = list(per_frame.values()) + result[label] = distribution(values) if values else None + if values: + worst = max(per_frame, key=per_frame.get) + result[label]["worst_frame_end_ns"] = frames[worst][0] + result["unassigned_scopes"].sort(key=lambda item: item["duration_ms"], reverse=True) + result["unassigned_scopes"] = result["unassigned_scopes"][:20] + return result + + +def main_frame_diagnostics(records, window): + # RuntimeUpdate contains non-loading work. Commit is nested within it and + # must be reported separately, rather than added to the update duration. + return frame_scope_diagnostics(records, window, "SceneLoad.Main.FrameWorkMs", ( + ("SceneLoad.Main.RuntimeUpdate", "runtime_update_ms"), + ("SceneLoad.Main.RuntimeGuardedCommit", "guarded_commit_ms"))) + + +def render_frame_diagnostics(records, window): + return frame_scope_diagnostics(records, window, "SceneLoad.Render.FrameWorkMs", ( + ("SceneLoad.Render.CookedUploadVisit", "cooked_upload_ms"), + ("SceneLoad.Render.EnvironmentPreparation", "environment_preparation_ms"))) + + +def window_occlusion(records, window): + """Return macOS window visibility transitions, including state at entry.""" + samples = sorted((timestamp, value) for kind, name, timestamp, value in records + if kind == "value" and name == "SceneLoad.Main.WindowOcclusionVisible" and timestamp <= window[1]) + if not samples: + return None + if any(value not in (0, 1, 2) for _, value in samples): + raise ValueError("invalid window occlusion observation") + preceding = [(timestamp, value) for timestamp, value in samples if timestamp <= window[0]] + return {"visible_at_start": preceding[-1][1] if preceding else None, + "changes": [{"timestamp_ns": timestamp, "visible": value} for timestamp, value in samples if window[0] < timestamp <= window[1]], + "scope": "macOS window occlusion state (0 occluded, 1 visible, 2 unknown); visibility does not prove drawable availability or timing cause"} + + +def render_outliers(records, window, steady_p95_ms, allowance_ms=8.0): + """Locate long intervals and overlapping render scopes without assigning cause.""" + threshold = steady_p95_ms + allowance_ms + intervals = [(timestamp, value) for kind, name, timestamp, value in records + if kind == "value" and name == "SceneLoad.Render.FrameIntervalMs" + and window[0] <= timestamp <= window[1] and value > threshold] + scopes = [(name, timestamp, duration) for kind, name, timestamp, duration in records + if kind == "scope_ns" and name.startswith("SceneLoad.Render.")] + platform_events = [(timestamp, duration) for kind, name, timestamp, duration in records + if kind == "scope_ns" and name == "SceneLoad.Main.PlatformEvents"] + worker_values = [(name, timestamp, value) for kind, name, timestamp, value in records + if kind == "value" and name.startswith("SceneLoad.RecordingWorker.")] + submit_values = [(name, timestamp, value) for kind, name, timestamp, value in records + if kind == "value" and name.startswith("SceneLoad.QueueSubmit.")] + details = [] + for end, interval_ms in sorted(intervals, key=lambda entry: entry[1], reverse=True)[:20]: + start = end - interval_ms * 1_000_000 + overlaps = [{"name": name, "end_ns": scope_end, "duration_ms": duration / 1_000_000} + for name, scope_end, duration in scopes + if scope_end >= start and scope_end - duration <= end] + overlaps.sort(key=lambda scope: scope["duration_ms"], reverse=True) + platform_overlaps = [{"end_ns": event_end, "duration_ms": duration / 1_000_000, + "overlap_ms": max(0, min(end, event_end) - max(start, event_end - duration)) / 1_000_000} + for event_end, duration in platform_events + if event_end >= start and event_end - duration <= end] + platform_overlaps.sort(key=lambda event: event["overlap_ms"], reverse=True) + worker_diagnostics = [{"name": name, "timestamp_ns": timestamp, "value_ms": value} + for name, timestamp, value in worker_values if start <= timestamp <= end] + worker_diagnostics.sort(key=lambda sample: sample["value_ms"], reverse=True) + submit_diagnostics = [{"name": name, "timestamp_ns": timestamp, "value_ms": value} + for name, timestamp, value in submit_values if start <= timestamp <= end] + details.append({"interval_end_ns": end, "interval_ms": interval_ms, "longest_overlapping_scopes": overlaps[:5], + "overlapping_platform_events": platform_overlaps[:5], "recording_worker_samples": worker_diagnostics[:20], + "queue_submit_samples": submit_diagnostics, "window_occlusion": window_occlusion(records, (start, end))}) + return {"threshold_ms": threshold, "count": len(intervals), "largest_intervals": details, + "note": "Overlap is diagnostic only; nested scopes are not additive and do not establish causation."} + + +def analyze(evidence, trace, minimum_steady_frames): + samples = evidence["samples"] + if len(samples) != 2 or [sample["generation"] for sample in samples] != [2, 3]: + raise ValueError("expected one initial load followed by one replacement") + first, replacement = samples + initial = (first["admission_ns"], first["first_submission_ns"]) + steady = (first["steady_start_ns"], first["steady_end_ns"]) + loading = (replacement["admission_ns"], replacement["first_submission_ns"]) + post = None + if any(key in replacement for key in ("post_steady_start_ns", "post_steady_end_ns", "post_steady_submitted_frames")): + if not all(key in replacement for key in ("post_steady_start_ns", "post_steady_end_ns", "post_steady_submitted_frames")): + raise ValueError("partial post-replacement steady window") + post = (replacement["post_steady_start_ns"], replacement["post_steady_end_ns"]) + if replacement["post_steady_submitted_frames"] < minimum_steady_frames or not (loading[1] <= post[0] < post[1]): + raise ValueError("invalid post-replacement steady window") + if first["steady_submitted_frames"] < minimum_steady_frames or not (0 < initial[0] < initial[1] <= steady[0] < steady[1] <= loading[0] < loading[1]): + raise ValueError("incomplete, overlapping or unordered timing windows") + records = list(capture_records(trace)) + timestamps = [timestamp for _, _, timestamp, _ in records] + if not timestamps or min(timestamps) > steady[0] or max(timestamps) < (post[1] if post else loading[1]): + raise ValueError("stage capture does not cover steady and replacement windows") + windows = {} + requested_windows = [("initial_load", initial), ("steady", steady), ("replacement", loading)] + if post: + requested_windows.append(("post_steady", post)) + for name, window in requested_windows: + stages = read_stages(trace, window) + required = {"SceneLoad.Main.FrameWorkMs", "SceneLoad.Main.FrameIntervalMs", "SceneLoad.Render.FrameIntervalMs"} + if not required.issubset(stages): + raise ValueError(f"missing frame observations in {name}") + windows[name] = {"start_ns": window[0], "end_ns": window[1], + "first_record_ns": min(timestamp for timestamp in timestamps if window[0] <= timestamp <= window[1]), + "window_occlusion": window_occlusion(records, window), + "main_frame_diagnostics": main_frame_diagnostics(records, window), + "render_frame_diagnostics": render_frame_diagnostics(records, window), + "stages": {stage: stages[stage] for stage in STAGES if stage in stages}} + if windows["steady"]["stages"]["SceneLoad.Render.FrameIntervalMs"]["count"] < minimum_steady_frames - 1: + raise ValueError("steady window does not contain the required render intervals") + if post and windows["post_steady"]["stages"]["SceneLoad.Render.FrameIntervalMs"]["count"] < minimum_steady_frames - 1: + raise ValueError("post-replacement steady window does not contain the required render intervals") + steady_p95 = windows["steady"]["stages"]["SceneLoad.Render.FrameIntervalMs"]["p95_ms"] + for name, window in (("initial_load", initial), ("replacement", loading)): + windows[name]["render_interval_outliers"] = render_outliers(records, window, steady_p95) + return {"scope": "diagnostic comparison; not matched-frame attribution or qualification", + "manifest_sha256": evidence["manifest_sha256"], + "filesystem_cache_conditions": evidence["filesystem_cache_conditions"], + "initial_trace_starts_after_admission_ns": max(0, min(timestamps) - initial[0]), + "initial_admission_marker_observed": sum(1 for kind, name, timestamp, _ in records + if kind == "scope_ns" and name == "SceneLoad.Main.BeforeInitialAdmission" and timestamp <= initial[0]) == 1, + "steady_submitted_frames": first["steady_submitted_frames"], "windows": windows} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--trace", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--minimum-steady-frames", type=int, default=120) + args = parser.parse_args() + if args.minimum_steady_frames < 2: + parser.error("--minimum-steady-frames must be at least two") + report = analyze(json.loads(args.evidence.read_text(encoding="utf-8")), args.trace, args.minimum_steady_frames) + with args.output.open("x", encoding="utf-8") as output: + json.dump(report, output, indent=2) + output.write("\n") + print(f"Wrote diagnostic timing report to {args.output}") + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, KeyError, TypeError) as error: + print(f"Runtime trace analysis failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/tests/scene/analyze_cooked_timing_samples.py b/tests/scene/analyze_cooked_timing_samples.py new file mode 100644 index 000000000..04c33b784 --- /dev/null +++ b/tests/scene/analyze_cooked_timing_samples.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Compare bounded fresh/replacement timing observations, not qualification.""" +import argparse +import json +from pathlib import Path +import re +import sys + +from validate_scene_lifecycle import distribution + + +def analyze(fresh, replacements, count=30): + if count < 1: + raise ValueError("sample count must be positive") + first = fresh["fresh_process_samples"] + same = replacements["samples"] + if len(first) != count or len(same) != count + 1 or replacements["reloads"] != count: + raise ValueError("fresh or replacement sample count is incomplete") + if [item["generation"] for item in first] != [2] * count or [item["generation"] for item in same] != list(range(2, count + 3)): + raise ValueError("unexpected scene generations") + if fresh["manifest_sha256"] != replacements["manifest_sha256"] or not re.fullmatch(r"[0-9a-f]{64}", fresh["manifest_sha256"]): + raise ValueError("timing runs do not use the same pinned cooked manifest") + if fresh["binary_sha256"] != replacements["binary_sha256"] or any(not re.fullmatch(r"[0-9a-f]{64}", fresh["binary_sha256"].get(key, "")) for key in ("runtime_sha256", "cooker_sha256")): + raise ValueError("timing runs do not use the same pinned binaries") + if fresh["host_system"] != replacements["host_system"] or fresh["host_machine"] != replacements["host_machine"]: + raise ValueError("timing runs do not identify the same host") + for key in ("hardware", "device", "shader_cache"): + if (key in fresh or key in replacements) and (key not in fresh or key not in replacements or fresh[key] != replacements[key]): + raise ValueError(f"timing runs do not identify the same {key}") + shaders = fresh.get("shader_cache") + if shaders is not None and (not isinstance(shaders, dict) or not re.fullmatch(r"[0-9a-f]{64}", str(shaders.get("sha256", ""))) or not isinstance(shaders.get("files"), int) or isinstance(shaders.get("files"), bool) or shaders["files"] < 1): + raise ValueError("invalid shader_cache identity") + hardware = fresh.get("hardware") + if hardware is not None and (not isinstance(hardware, dict) or not isinstance(hardware.get("cpu"), str) or not hardware["cpu"].strip() or not isinstance(hardware.get("physical_memory_bytes"), int) or isinstance(hardware.get("physical_memory_bytes"), bool) or hardware["physical_memory_bytes"] <= 0): + raise ValueError("invalid hardware identity") + device = fresh.get("device") + if device is not None and (not isinstance(device, dict) or not isinstance(device.get("gpu"), str) or not device["gpu"].strip() or not isinstance(device.get("driver_path"), str) or not device["driver_path"].strip() or not re.fullmatch(r"[0-9a-f]{64}", str(device.get("driver_sha256", "")))): + raise ValueError("invalid device identity") + if fresh["validation_states"] != [True] * count or replacements["validation_states"] != [True]: + raise ValueError("validation was not confirmed for every process") + if fresh["source_audit_enabled"] is not True or replacements["source_audit_enabled"] is not True: + raise ValueError("source-independence audit was not enabled for both runs") + if fresh["gpu_completion_observed"] is not True or replacements["gpu_completion_observed"] is not True: + raise ValueError("GPU completion observation was not enabled for both runs") + viewports = fresh["timing_viewports"] + replacements["timing_viewports"] + if len(viewports) != count + 1 or any(not isinstance(view, dict) or view.get("resize_after_baseline") is not False for view in viewports): + raise ValueError("viewport evidence is missing or includes a resize") + for view in viewports: + if any(not isinstance(view.get(key), list) or len(view[key]) != 2 or any(not isinstance(value, int) or isinstance(value, bool) or value <= 0 for value in view[key]) for key in ("scene_viewport", "window", "framebuffer")): + raise ValueError("invalid viewport dimensions") + if any(view != viewports[0] for view in viewports[1:]): + raise ValueError("timing runs do not use matching viewport dimensions") + if any(not isinstance(item["filesystem_cache_conditions"], str) or not item["filesystem_cache_conditions"].strip() for item in (fresh, replacements)): + raise ValueError("filesystem cache conditions are missing") + + def milliseconds(items): + result = [] + for item in items: + admission, submitted, elapsed = (item[key] for key in ("admission_ns", "first_submission_ns", "admission_to_submission_ns")) + if any(isinstance(value, bool) or not isinstance(value, int) for value in (admission, submitted, elapsed)) or admission <= 0 or submitted <= admission or elapsed != submitted - admission: + raise ValueError("invalid admission-to-submission sample") + result.append(elapsed / 1_000_000) + return result + + def completion_milliseconds(items): + result = [] + for item in items: + observed, timeline, elapsed = (item[key] for key in ("render_completion_observed_ns", "render_timeline_value", "admission_to_completion_observed_ns")) + if any(isinstance(value, bool) or not isinstance(value, int) for value in (observed, timeline, elapsed)) or timeline <= 0 or observed < item["first_submission_ns"] or elapsed != observed - item["admission_ns"]: + raise ValueError("invalid observed render completion") + result.append(elapsed / 1_000_000) + return result + + return {"scope": "diagnostic accepted-submission and observed GPU-completion timing; not presentation or qualification", + "manifest_sha256": fresh["manifest_sha256"], "binary_sha256": fresh["binary_sha256"], + "host_system": fresh["host_system"], "host_machine": fresh["host_machine"], + **{key: fresh[key] for key in ("hardware", "device", "shader_cache") if key in fresh}, + "viewport": viewports[0], + "cache_conditions": {"fresh": fresh["filesystem_cache_conditions"], "replacement": replacements["filesystem_cache_conditions"]}, + "fresh_process_loads": distribution(milliseconds(first)), + "same_process_initial_load": distribution(milliseconds(same[:1])), + "same_process_replacements": distribution(milliseconds(same[1:])), + "same_process_first_replacement": distribution(milliseconds(same[1:2])), + "same_process_later_replacements": distribution(milliseconds(same[2:])) if count > 1 else None, + "fresh_process_completion_observed": distribution(completion_milliseconds(first)), + "same_process_initial_completion_observed": distribution(completion_milliseconds(same[:1])), + "same_process_replacements_completion_observed": distribution(completion_milliseconds(same[1:])), + "same_process_first_replacement_completion_observed": distribution(completion_milliseconds(same[1:2])), + "same_process_later_replacements_completion_observed": distribution(completion_milliseconds(same[2:])) if count > 1 else None, + "note": "Fresh process does not imply cold disk I/O. GPU completion is an upper-bound polling observation, not presentation."} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--fresh", required=True, type=Path) + parser.add_argument("--replacements", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--count", type=int, default=30) + args = parser.parse_args() + report = analyze(json.loads(args.fresh.read_text()), json.loads(args.replacements.read_text()), args.count) + with args.output.open("x", encoding="utf-8") as target: + json.dump(report, target, indent=2) + target.write("\n") + print(f"Wrote timing diagnostics to {args.output}") + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, KeyError, TypeError) as error: + print(f"Cooked timing analysis failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/Scripts/compare_scene_capture.py b/tests/scene/compare_scene_capture.py similarity index 92% rename from Scripts/compare_scene_capture.py rename to tests/scene/compare_scene_capture.py index 2cf087842..22cc3d010 100644 --- a/Scripts/compare_scene_capture.py +++ b/tests/scene/compare_scene_capture.py @@ -9,6 +9,7 @@ """ import argparse +import gzip import hashlib import json from pathlib import Path @@ -24,15 +25,24 @@ def validate_capture_log(path): data = source.read(MAX_LOG_BYTES + 1) if not data or len(data) > MAX_LOG_BYTES: raise ValueError("missing or oversized capture validation log") - text = data.decode("utf-8", errors="strict").lower() + validate_capture_log_text(data.decode("utf-8", errors="strict")) + + +def validate_capture_log_text(text): + """Validate a complete, already reconstructed process log.""" + if not text or len(text.encode("utf-8")) > MAX_LOG_BYTES: + raise ValueError("missing or oversized capture validation log") + text = text.lower() if any(marker in text for marker in ("[error]", "vuid-", "vk_not_ready", "validation error")): - raise ValueError(f"capture validation errors in {path}") + raise ValueError("capture validation errors in process log") if "engine destroyed" not in text: raise ValueError("capture log does not show completed engine shutdown") def read_ppm(path): - with Path(path).open("rb") as source: + path = Path(path) + opener = gzip.open if path.suffix == ".gz" else Path.open + with opener(path, "rb") as source: data = source.read(MAX_BYTES + 1) if len(data) > MAX_BYTES: raise ValueError("capture exceeds the bounded image profile") diff --git a/tests/scene/cooked_capacity_probe.py b/tests/scene/cooked_capacity_probe.py new file mode 100644 index 000000000..f9022356d --- /dev/null +++ b/tests/scene/cooked_capacity_probe.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 +"""Generate a many-instance cooked fixture and run it in standalone Vulkan. + +This is capacity evidence for a synthetic shared triangle/material, not a +qualification profile for arbitrary complex scenes. The source fixture is +removed before launch. A nonzero exit reports failure without claiming that +the requested capacity is supported. +""" +import argparse +import hashlib +import json +from pathlib import Path +import platform +import re +import shutil +import subprocess +import sys +import tempfile + +from analyze_cooked_runtime_trace import main_frame_diagnostics, render_outliers +from cooked_scene_smoke import (observed_timing_viewport, read_cancellation_timings, read_console_log, + read_lifecycle_observations, read_process_logs, require_clean_stress_log, + require_clean_vulkan_console, require_stable_stress_window, require_zero_source_audit, + sha256_file, validate_stress_sequence, validation_layer_status_text) +from validate_scene_lifecycle import capture_records, read_stages + + +SCENE = "10000000-0000-0000-0000-000000000001" +ALTERNATE_SCENE = "90000000-0000-0000-0000-000000000009" +MATERIAL = "40000000-0000-0000-0000-000000000004" +ALTERNATE_MATERIAL = "b0000000-0000-0000-0000-00000000000b" +MAX_DRAWS = 8192 + + +def validate_unique_material_manifest(manifest): + scenes = {scene["uuid"]: scene for scene in manifest["scenes"]} + assets = {asset["uuid"]: asset for asset in manifest["assets"] if asset["kind"] == "material"} + if SCENE not in scenes or ALTERNATE_SCENE not in scenes or MATERIAL not in assets or ALTERNATE_MATERIAL not in assets: + raise ValueError("distinct-material package is missing expected scenes or materials") + roots = lambda scene: {item["uuid"] for item in scene["roots"] if item["kind"] == "material"} + if MATERIAL not in roots(scenes[SCENE]) or ALTERNATE_MATERIAL not in roots(scenes[ALTERNATE_SCENE]): + raise ValueError("distinct-material package has unexpected scene roots") + if assets[MATERIAL]["object"]["digest"] == assets[ALTERNATE_MATERIAL]["object"]["digest"]: + raise ValueError("distinct materials do not have distinct cooked objects") + + +def summarize_initial_trace(trace, sample): + records = list(capture_records(trace)) + timestamps = [timestamp for _, _, timestamp, _ in records] + start, end = sample["admission_ns"], sample["first_submission_ns"] + if not timestamps or start >= end or max(timestamps) < end: + raise ValueError("stage capture does not cover the first submission") + markers = [timestamp for kind, name, timestamp, _ in records + if kind == "scope_ns" and name == "SceneLoad.Main.BeforeInitialAdmission" and timestamp <= start] + return {"scope": "diagnostic only; partial initial coverage cannot prove a frame budget", + "starts_after_admission_ns": max(0, min(timestamps) - start), + "admission_marker_observed": len(markers) == 1, + "main_frame_diagnostics": main_frame_diagnostics(records, (start, end)), + "stages": read_stages(trace, (start, end))} + + +def validate_publications(log, expect_refusal, replacement_refusal, steady_replacement=False): + publications = [(int(generation), scene) for generation, scene in re.findall( + r"\[CookedRuntime\] Published cooked scene generation (\d+) scene=([0-9a-f-]+)", log)] + expected = ([] if expect_refusal else [(2, SCENE), (3, SCENE)] if replacement_refusal + else [(2, SCENE), (3, ALTERNATE_SCENE)] if steady_replacement else [(2, SCENE)]) + if publications != expected: + raise ValueError("capacity run published an unexpected scene or generation") + if replacement_refusal and re.search(r"\[SkyEnvironment\].*using the fallback", log): + raise ValueError("capacity failure caused an environment fallback") + return [generation for generation, _ in publications] + + +def make_texture_variants(directory, count): + """Create distinct one-pixel textures using the fixture's BMP layout.""" + if count < 1: + raise ValueError("texture count must be positive") + source = (directory / "pixel.bmp").read_bytes() + metadata = json.loads((directory / "pixel.bmp.meta").read_text()) + base = "50000000-0000-0000-0000-000000000005" + if len(source) != 58 or source[:2] != b"BM" or int.from_bytes(source[10:14], "little") != 54 or metadata.get("uuid") != base: + raise ValueError("unexpected capacity texture fixture") + identities = [(base, "assets/pixel.bmp")] + for index in range(1, count): + identity = f"c0000000-0000-4000-8000-{index:012x}" + name = f"capacity-texture-{index:05d}.bmp" + pixels = bytearray(source) + pixels[54:57] = index.to_bytes(3, "little") + (directory / name).write_bytes(pixels) + (directory / (name + ".meta")).write_text(json.dumps(dict(metadata, uuid=identity), sort_keys=True) + "\n") + identities.append((identity, "assets/" + name)) + return identities + + +def make_material_variants(directory, count, textures=None): + """Create distinct material objects using the requested texture versions.""" + if count < 1: + raise ValueError("material count must be positive") + source = json.loads((directory / "material.zematerial").read_text()) + meta = json.loads((directory / "material.zematerial.meta").read_text()) + if source.get("uuid") != MATERIAL or meta.get("uuid") != MATERIAL: + raise ValueError("unexpected capacity fixture material identity") + identities = [MATERIAL] + for index in range(1, count): + identity = f"a0000000-0000-4000-8000-{index:012x}" + material = dict(source, uuid=identity, name=f"Capacity material {index}") + if textures: + texture_uuid, texture_path = textures[index % len(textures)] + material["textures"] = dict(source["textures"], albedo={"uuid": texture_uuid, "path": texture_path}) + metadata = dict(meta, uuid=identity) + path = directory / f"capacity-material-{index:05d}.zematerial" + path.write_text(json.dumps(material, sort_keys=True) + "\n") + path.with_suffix(path.suffix + ".meta").write_text(json.dumps(metadata, sort_keys=True) + "\n") + identities.append(identity) + return identities + + +def append_instances(scene, draws, materials=None): + """The base fixture already contains one mesh and one camera.""" + if not 1 <= draws <= MAX_DRAWS + 1: + raise ValueError("draw count must be between 1 and the limit plus one") + original = scene.read_bytes() + if not original.endswith(b"\n"): + raise ValueError("fixture scene is not newline terminated") + materials = materials or [MATERIAL] + if materials[0] != MATERIAL or len(materials) > draws or len(set(materials)) != len(materials): + raise ValueError("invalid fixture material identities") + with scene.open("ab") as target: + for index in range(1, draws): + target.write(f''' - "uuid": "d0000000-0000-4000-8000-{index:012x}" + "components": + "TransformComponent": + "schema_version": 1 + "fields": + "position": [0, 0, 0] + "rotation": [0, 0, 0] + "scale": [1, 1, 1] + "MeshComponent": + "schema_version": 1 + "fields": + "mesh_uuid": "30000000-0000-0000-0000-000000000003" + "MaterialComponent": + "schema_version": 1 + "fields": + "material_uuid": "{materials[index % len(materials)]}" +'''.encode()) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cooker", required=True, type=Path) + parser.add_argument("--obelisk", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--draws", type=int, default=MAX_DRAWS) + parser.add_argument("--materials", type=int, default=1, help="number of distinct material versions sharing the fixture texture") + parser.add_argument("--textures", type=int, default=1, help="number of distinct one-pixel textures used by the materials") + parser.add_argument("--expect-refusal", action="store_true", help="expect the limit-plus-one fixture to fail before publication") + parser.add_argument("--replacement-refusal", action="store_true", help="reject a limit-plus-one replacement and retry the active scene") + parser.add_argument("--texture-overlap-refusal", action="store_true", help="reject a larger texture replacement and retry the smaller active scene") + parser.add_argument("--active-materials", type=int, default=0, help="material count in the initial scene for texture-overlap refusal") + parser.add_argument("--stage-trace", action="store_true", help="capture diagnostic load stages at a fixed logical window") + parser.add_argument("--steady-frames", type=int, default=0, help="record settled standalone frames after initial load (requires --stage-trace)") + parser.add_argument("--steady-replacement", action="store_true", help="replace the settled scene with an equivalent capacity scene") + parser.add_argument("--unique-replacement-material", action="store_true", help="use a distinct material version in the replacement scene") + parser.add_argument("--lifecycle-cycles", type=int, default=0, help="alternate cancellations and replacements of the capacity scene") + parser.add_argument("--timeout", type=float, default=600) + args = parser.parse_args() + cooker, obelisk = args.cooker.resolve(strict=True), args.obelisk.resolve(strict=True) + binary_hashes = {"cooker_sha256": sha256_file(cooker), "runtime_sha256": sha256_file(obelisk)} + if (args.timeout <= 0 or not 1 <= args.draws <= MAX_DRAWS + 1 or not 1 <= args.materials <= args.draws or not 1 <= args.textures <= args.materials or args.expect_refusal and args.replacement_refusal + or (args.draws == MAX_DRAWS + 1) != (args.expect_refusal or args.replacement_refusal) + or not 0 <= args.steady_frames <= 600 + or (args.steady_frames and (not args.stage_trace or args.expect_refusal or args.replacement_refusal)) + or (args.steady_replacement and not args.steady_frames) + or (args.unique_replacement_material and not args.steady_replacement) + or (args.materials > 1 and (args.unique_replacement_material or args.replacement_refusal)) + or (args.texture_overlap_refusal and (args.expect_refusal or args.replacement_refusal or args.steady_replacement or args.lifecycle_cycles + or not 1 <= args.active_materials < args.textures or args.textures != args.materials)) + or (args.active_materials and not args.texture_overlap_refusal) + or args.lifecycle_cycles < 0 + or (args.lifecycle_cycles and (args.lifecycle_cycles < 4 or args.lifecycle_cycles % 4 + or args.expect_refusal or args.replacement_refusal or args.stage_trace))): + parser.error("invalid capacity probe options; lifecycle cycles require a multiple of four and cannot be combined with refusal or stage-trace modes") + output = args.output.resolve(strict=True) + if not output.is_dir() or any(output.iterdir()): + parser.error("output must be an existing empty directory") + with tempfile.TemporaryDirectory(prefix="renderer-cooked-capacity-") as temporary: + root = Path(temporary) + authoring = root / "authoring" + authoring.mkdir() + fixture_mode = "write-switch-unique-material" if args.unique_replacement_material else "write-camera" + subprocess.run([str(cooker), fixture_mode, str(authoring)], check=True, timeout=args.timeout) + scene = authoring / "scenes" / "scène_東京.scene" + fixture = authoring / "scenes" / "fixture.zescene" + shutil.copyfile(scene, fixture) + capacity_scene = fixture + if args.replacement_refusal: + capacity_scene = authoring / "scenes" / "alternate.zescene" + original = fixture.read_bytes() + if original.count(SCENE.encode()) != 1: + raise ValueError("fixture does not contain a unique scene UUID") + capacity_scene.write_bytes(original.replace(SCENE.encode(), ALTERNATE_SCENE.encode(), 1)) + textures = make_texture_variants(authoring / "assets", args.textures) + materials = make_material_variants(authoring / "assets", args.materials, textures) + append_instances(capacity_scene, args.draws, materials[:args.active_materials] if args.texture_overlap_refusal else materials) + if args.texture_overlap_refusal: + original = (authoring / "scenes" / "scène_東京.scene").read_bytes() + if original.count(SCENE.encode()) != 1: + raise ValueError("fixture does not contain a unique scene UUID") + alternate = authoring / "scenes" / "alternate.zescene" + alternate.write_bytes(original.replace(SCENE.encode(), ALTERNATE_SCENE.encode(), 1)) + append_instances(alternate, args.draws, materials) + if args.steady_replacement or args.lifecycle_cycles: + original = fixture.read_bytes() + if original.count(SCENE.encode()) != 1: + raise ValueError("fixture does not contain a unique scene UUID") + alternate = original.replace(SCENE.encode(), ALTERNATE_SCENE.encode(), 1) + if args.unique_replacement_material: + if alternate.count(MATERIAL.encode()) != args.draws: + raise ValueError("fixture does not contain one material reference per draw") + alternate = alternate.replace(MATERIAL.encode(), ALTERNATE_MATERIAL.encode()) + (authoring / "scenes" / "alternate.zescene").write_bytes(alternate) + source_digest = hashlib.sha256(capacity_scene.read_bytes()).hexdigest() + alternate_digest = hashlib.sha256((authoring / "scenes" / "alternate.zescene").read_bytes()).hexdigest() if args.texture_overlap_refusal else None + # The regular fixture's recook mode deliberately caps decoded scene + # memory at 128 KiB. Use the offline project cook with a larger bounded + # working budget for capacity input, just as the real project path does. + config = {"projectName": "CookedCapacityFixture", "workingSpace": ".", "sceneDir": "scenes", + "assetDirs": {key: "assets" for key in ("textureDir", "soundDir", "meshDir", "materialDir", "spriteDir", "environmentMapDir")}, + "sceneList": [{"name": "fixture", "isDefault": True}]} + if args.replacement_refusal or args.texture_overlap_refusal or args.steady_replacement or args.lifecycle_cycles: + config["sceneList"].append({"name": "alternate", "isDefault": False}) + (authoring / "project.json").write_text(json.dumps(config), encoding="utf-8") + cooked = root / "cooked" + cooked.mkdir() + subprocess.run([str(cooker), "smoke-offline", str(authoring / "project.json"), str(cooked), "805306368"], check=True, timeout=args.timeout) + package = root / "package" + shutil.copytree(cooked / "output" / "desktop-vulkan-le64", package) + manifest = (package / "manifest.json").read_bytes() + manifest_data = json.loads(manifest) + material_assets = {item["uuid"]: item for item in manifest_data["assets"] if item["kind"] == "material"} + if not set(materials).issubset(material_assets) or len({material_assets[identity]["object"]["digest"] for identity in materials}) != len(materials): + raise ValueError("capacity package lacks distinct material versions") + texture_assets = {item["uuid"]: item for item in manifest_data["assets"] if item["kind"] == "texture"} + texture_ids = [identity for identity, _ in textures] + if not set(texture_ids).issubset(texture_assets) or len({texture_assets[identity]["object"]["digest"] for identity in texture_ids}) != len(texture_ids): + raise ValueError("capacity package lacks distinct texture versions") + expected_scene = ALTERNATE_SCENE if args.replacement_refusal or args.texture_overlap_refusal else SCENE + roots = {item["uuid"] for scene in manifest_data["scenes"] if scene["uuid"] == expected_scene + for item in scene["roots"] if item["kind"] == "material"} + if not set(materials).issubset(roots): + raise ValueError("capacity scene does not reference every requested material") + if args.texture_overlap_refusal: + active_roots = {item["uuid"] for scene in manifest_data["scenes"] if scene["uuid"] == SCENE + for item in scene["roots"] if item["kind"] == "material"} + if active_roots != set(materials[:args.active_materials]): + raise ValueError("active scene material set differs from the refusal fixture") + if args.unique_replacement_material: + validate_unique_material_manifest(manifest_data) + (output / "manifest.json").write_bytes(manifest) + shutil.rmtree(authoring) + shutil.rmtree(cooked) + logs = output / "runtime-log" + logs.mkdir() + command = [str(obelisk), "--cookedStandalone", "--cookedPackage", str(package), "--cookedScene", SCENE, + "--cookedExitAfterFrame", "--cookedExpectedDraws", str(1 if args.replacement_refusal else args.draws), "--cookedAuditSource", "--logDirectory", str(logs)] + if args.stage_trace: + command.extend(["--cookedFixedWindow", "--cookedStageTrace", str(output / "stage.csv")]) + if args.steady_frames: + command.extend(["--cookedSteadyFrames", str(args.steady_frames)]) + if args.replacement_refusal or args.texture_overlap_refusal: + command.extend(["--cookedReloads", "1", "--cookedAlternateScene", ALTERNATE_SCENE, "--cookedRetryFailure"]) + elif args.steady_replacement: + command.extend(["--cookedReloads", "1", "--cookedAlternateScene", ALTERNATE_SCENE]) + elif args.lifecycle_cycles: + command.extend(["--cookedReloads", str(args.lifecycle_cycles), "--cookedAlternateScene", ALTERNATE_SCENE, + "--cookedLifecycleStress", "--cookedFixedWindow"]) + stdout_path, stderr_path = logs / "stdout.log", logs / "stderr.log" + with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + completed = subprocess.run(command, timeout=args.timeout, stdout=stdout, stderr=stderr) + log = read_process_logs(logs) + require_clean_vulkan_console(read_console_log(stdout_path)) + require_clean_vulkan_console(read_console_log(stderr_path)) + if sha256_file(obelisk) != binary_hashes["runtime_sha256"]: + raise RuntimeError("runtime executable changed during the capacity observation") + valid_refusal = (args.expect_refusal and completed.returncode == 2 + and "complete scene exceeds the instance budget" in log + and "Published cooked scene generation" not in log) + valid_render = (not args.expect_refusal and completed.returncode == 0 + and not args.replacement_refusal and not args.texture_overlap_refusal and f"[CookedRuntime] Submitted {args.draws} draws" in log) + valid_replacement = (args.replacement_refusal and completed.returncode == 0 + and "complete scene exceeds the instance budget" in log + and "Failed replacement outcome retained and acknowledged" in log + and "Replacement failed; active scene generation 2 retained" in log + and "Active scene submitted after failure; retry admitted" in log + and f"Published cooked scene generation 3 scene={SCENE}" in log + and "[CookedRuntime] Submitted 1 draws" in log) + valid_texture_refusal = (args.texture_overlap_refusal and completed.returncode == 0 + and "texture allocation status=-10" in log + and "Failed replacement outcome retained and acknowledged" in log + and "Replacement failed; active scene generation 2 retained" in log + and "Active scene submitted after failure; retry admitted" in log + and f"Published cooked scene generation 3 scene={SCENE}" in log + and f"[CookedRuntime] Submitted {args.draws} draws" in log) + require_zero_source_audit(log) + if not (valid_refusal or valid_render or valid_replacement or valid_texture_refusal) or not validation_layer_status_text(log) or "Engine destroyed" not in log or any(marker in log.lower() for marker in ("vuid-", "validation error", "vk_not_ready")): + raise RuntimeError(f"capacity fixture did not render and shut down cleanly (exit {completed.returncode}); inspect {logs}") + cancellation_timings = [] + if args.lifecycle_cycles: + validate_stress_sequence(log, args.lifecycle_cycles) + cancellation_timings = read_cancellation_timings(log, args.lifecycle_cycles) + require_stable_stress_window(log) + require_clean_stress_log(log) + cancelled = args.lifecycle_cycles // 2 + if (f"[CookedStress] admitted={args.lifecycle_cycles} cancelled={cancelled} committed={cancelled}" not in log + or log.count("Cancelled worker outcome acknowledged before cleanup") != cancelled // 2 + or log.count("Cancelled upload outcome acknowledged before cleanup") != cancelled // 2): + raise ValueError("capacity lifecycle run did not observe every cancellation outcome") + generations = list(range(2, cancelled + 3)) + else: + generations = validate_publications(log, args.expect_refusal, args.replacement_refusal or args.texture_overlap_refusal, args.steady_replacement) + samples = read_lifecycle_observations(log, generations, require_gpu_budget=True) if generations else [] + if args.lifecycle_cycles and any(sample[field] for sample in samples + for field in ("cache_closures", "cache_leases", "cache_retiring", "cache_asset_versions")): + raise ValueError("capacity lifecycle run retained an unleased cooked closure") + report = {"scope": "synthetic shared-mesh cooked capacity observation; not a calibrated profile", + "draws": args.draws, "entities_including_camera": args.draws + 1, "distinct_materials": args.materials, "distinct_textures": args.textures, + "limit_plus_one_refused": args.expect_refusal or args.replacement_refusal, + "active_scene_retained_and_retried": args.replacement_refusal or args.texture_overlap_refusal, + "texture_overlap_refused_and_retried": args.texture_overlap_refusal, + "active_materials": args.active_materials if args.texture_overlap_refusal else args.materials, + "unique_replacement_material": args.unique_replacement_material, + "scene_sha256": source_digest, "manifest_sha256": hashlib.sha256(manifest).hexdigest(), + "alternate_scene_sha256": alternate_digest, + "binary_sha256": binary_hashes, "host_system": platform.platform(), "host_machine": platform.machine(), + "filesystem_cache_conditions": "uncontrolled", "samples": samples, + "lifecycle_cycles": args.lifecycle_cycles, "cancellation_timings": cancellation_timings, + "validation_enabled": True, "source_audit_zero": True} + if args.stage_trace and samples: + report["initial_load_trace"] = summarize_initial_trace(output / "stage.csv", samples[0]) + if args.steady_frames: + windows = [tuple(map(int, row)) for row in re.findall(r"\[CookedSteady\] generation=(\d+) start_ns=(\d+) end_ns=(\d+) submitted_frames=(\d+)", log)] + viewport = observed_timing_viewport(log) + if len(windows) != 1 or not viewport or viewport["resize_after_baseline"]: + raise ValueError("missing steady window or viewport changed during observation") + generation, start, end, frames = windows[0] + if generation != generations[0] or frames < args.steady_frames or start < samples[0]["first_submission_ns"] or end <= start: + raise ValueError("invalid steady-rendering window") + trace = output / "stage.csv" + steady = read_stages(trace, (start, end)) + intervals = steady.get("SceneLoad.Render.FrameIntervalMs") + if not intervals or intervals["count"] < args.steady_frames - 1: + raise ValueError("steady trace lacks the required render intervals") + trace_records = list(capture_records(trace)) + report["steady_render"] = {"start_ns": start, "end_ns": end, "submitted_frames": frames, + "viewport": viewport, "intervals": intervals, + "steady_interval_outliers": render_outliers(trace_records, (start, end), intervals["p95_ms"]), + "initial_interval_outliers": render_outliers(trace_records, + (samples[0]["admission_ns"], samples[0]["first_submission_ns"]), intervals["p95_ms"]), + "scope": "diagnostic comparison; initial bootstrap and steady scene differ, so not matched-frame qualification"} + if args.steady_replacement: + if len(samples) != 2 or end > samples[1]["admission_ns"]: + raise ValueError("replacement overlaps the steady-rendering window") + replacement = (samples[1]["admission_ns"], samples[1]["first_submission_ns"]) + report["steady_render"]["replacement_interval_outliers"] = render_outliers( + trace_records, replacement, intervals["p95_ms"]) + report["steady_render"]["replacement_main_frame_diagnostics"] = main_frame_diagnostics(trace_records, replacement) + report["steady_render"]["replacement_stages"] = read_stages(trace, replacement) + (output / "capacity.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2)) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as error: + print(f"Cooked capacity probe failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/tests/scene/cooked_capture_identity.py b/tests/scene/cooked_capture_identity.py new file mode 100644 index 000000000..7ec0c68a6 --- /dev/null +++ b/tests/scene/cooked_capture_identity.py @@ -0,0 +1,29 @@ +"""Identify the shader cache and Vulkan device used by diagnostic captures.""" +import hashlib +from pathlib import Path +import re + +from cooked_scene_smoke import sha256_file + + +def shader_identity(directory): + directory = Path(directory) + paths = sorted(directory.rglob("*.spv")) + if not paths or any(path.is_symlink() for path in directory.rglob("*")) or any(not path.is_file() for path in paths): + raise ValueError("shader cache is missing or contains non-file shader entries") + digest = hashlib.sha256() + for path in paths: + relative = path.relative_to(directory).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(bytes.fromhex(sha256_file(path))) + return {"sha256": digest.hexdigest(), "files": len(paths)} + + +def device_identity(log): + devices = re.findall(r"Created VkDevice to run on GPU (.+?) with the following", log) + drivers = re.findall(r'Using "[^"]+" with driver: "([^"]+)"', log) + if len(devices) != 1 or len(drivers) != 1: + return None + driver = Path(drivers[0]).resolve(strict=True) + return {"gpu": devices[0], "driver_path": str(driver), "driver_sha256": sha256_file(driver)} diff --git a/tests/scene/cooked_scene_parity.py b/tests/scene/cooked_scene_parity.py new file mode 100644 index 000000000..f08ebb4c2 --- /dev/null +++ b/tests/scene/cooked_scene_parity.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Compare source and cooked renders of one generated HDR/mesh fixture. + +Requires a desktop Vulkan session and built SceneCookProcess and Obelisk. +The generated fixture is a semantic smoke test, not a reviewed golden or the +SampleProject qualification fixture. The output directory must be empty. +""" +import argparse +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +from compare_scene_capture import compare_region, read_ppm, validate_capture_log_text +from cooked_scene_smoke import read_process_logs, require_zero_source_audit +from sampleproject_cooked_runtime import run_capture, validation_layer_status_text + + +SCENE = "10000000-0000-0000-0000-000000000001" +ENVIRONMENT = "60000000-0000-0000-0000-000000000006" + + +def run(command, timeout): + subprocess.run([str(arg) for arg in command], check=True, timeout=timeout) + + +def compare_transparency(opaque, blended): + """Require a visible region between the opaque triangle and background.""" + width, height, opaque_pixels = opaque + if blended[:2] != (width, height): + raise ValueError("opaque and transparent reference dimensions differ") + blend_pixels = blended[2] + background = opaque_pixels[:3] + intermediate = sum(all(opaque_pixels[index + channel] + 8 < blend_pixels[index + channel] < background[channel] - 8 + for channel in range(3)) + for index in range(0, len(opaque_pixels), 3)) + return {"intermediate_pixels": intermediate, "minimum_pixels": (width * height + 499) // 500, + "passed": intermediate * 500 >= width * height} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cooker", required=True, type=Path) + parser.add_argument("--obelisk", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--timeout", type=float, default=120) + variant = parser.add_mutually_exclusive_group() + variant.add_argument("--blend", action="store_true", help="use the transparent material fixture") + variant.add_argument("--hierarchy", action="store_true", help="place the mesh under a translated parent") + parser.add_argument("--flat-reference", type=Path, help="compare the hierarchy capture with the equivalent flat source capture") + parser.add_argument("--opaque-reference", type=Path, help="compare a blend capture with the matching opaque source capture") + parser.add_argument("--require-validation", action="store_true", help="require Khronos validation for both captures") + parser.add_argument("--standalone", action="store_true", help="use the authored camera and source-independent standalone runtime") + args = parser.parse_args() + cooker = args.cooker.resolve(strict=True) + obelisk = args.obelisk.resolve(strict=True) + output = args.output.resolve(strict=True) + if args.timeout <= 0 or not output.is_dir() or any(output.iterdir()): + parser.error("--output must be an existing empty directory and --timeout positive") + if args.flat_reference and not args.hierarchy: + parser.error("--flat-reference requires --hierarchy") + if args.opaque_reference and not args.blend: + parser.error("--opaque-reference requires --blend") + source_log, cooked_log = output / "source-log", output / "cooked-log" + source_log.mkdir() + cooked_log.mkdir() + source_image, cooked_image = output / "source.ppm", output / "cooked.ppm" + + with tempfile.TemporaryDirectory(prefix="renderer-parity-") as temporary: + root = Path(temporary) + baseline = root / "baseline" + baseline.mkdir() + mode = "write-blend" if args.blend else "write-hierarchy" if args.hierarchy else "write" + if args.standalone: + mode = "write-camera" if mode == "write" else mode + "-camera" + run([cooker, mode, baseline], args.timeout) + source = root / "source" + source.mkdir() + for part in ("assets", "_cache"): + shutil.copytree(baseline / part, source / part) + (source / "scenes").mkdir() + shutil.copyfile(baseline / "scenes" / "scène_東京.scene", source / "scenes" / "fixture.zescene") + # The cooker fixture's 2x1 HDR tests serialization but cannot be + # imported by the editor. Use a valid constant 4x2 source and have the + # real importer produce the exact artifact later consumed by both paths. + (source / "assets" / "sky.hdr").write_bytes( + b"#?RADIANCE\nFORMAT=32-bit_rle_rgbe\n\n-Y 2 +X 4\n" + b"\x80\x80\x80\x81" * 8) + config = {"projectName": "ParityFixture", "workingSpace": ".", "sceneDir": "scenes", + "assetDirs": {key: "assets" for key in ("textureDir", "soundDir", "meshDir", "materialDir", "spriteDir", "environmentMapDir")}, + "sceneList": [{"name": "fixture", "isDefault": True}]} + (source / "project.json").write_text(json.dumps(config), encoding="utf-8") + source_view = ["--cookedRuntimeView", "--cookedFixedWindow"] if args.standalone else [] + run_capture([str(arg) for arg in [obelisk, "--launchEditor", "true", "--projectConfigFile", source / "project.json", + "--sourceCapture", source_image, "--sourceImport", "/assets/sky.hdr", + "--cookedExpectedDraws", "1", "--logDirectory", source_log, *source_view]], source_log, args.timeout) + source_text = read_process_logs(source_log) + validate_capture_log_text(source_text) + source_validation = validation_layer_status_text(source_text) + if args.require_validation and not source_validation: + raise ValueError("source capture did not enable Khronos Vulkan validation") + + cooking = root / "cooking" + cooking.mkdir() + for part in ("assets", "_cache"): + shutil.copytree(source / part, cooking / part) + for part in ("scenes", "output", "scratch"): + shutil.copytree(baseline / part, cooking / part) + meta_path = cooking / "assets" / "sky.hdr.meta" + meta = json.loads(meta_path.read_text(encoding="utf-8")) + meta["artifact_path"] = f"envmaps/{ENVIRONMENT}.zenvmap" + meta_path.write_text(json.dumps(meta), encoding="utf-8") + run([cooker, "recook", cooking], args.timeout) + package = root / "package" + shutil.copytree(cooking / "output" / "desktop-vulkan-le64", package) + # The consumer cannot reach the generated source tree or its registry. + shutil.rmtree(source) + shutil.rmtree(cooking) + shutil.rmtree(baseline) + launch = ["--cookedStandalone", "--cookedAuditSource", "--cookedFixedWindow"] if args.standalone else ["--launchEditor", "true"] + run_capture([str(arg) for arg in [obelisk, *launch, "--cookedPackage", package, + "--cookedScene", SCENE, "--cookedExitAfterFrame", "--cookedExpectedDraws", "1", + "--cookedCapture", cooked_image, "--logDirectory", cooked_log]], cooked_log, args.timeout) + cooked_text = read_process_logs(cooked_log) + validate_capture_log_text(cooked_text) + if args.standalone: + require_zero_source_audit(cooked_text) + cooked_validation = validation_layer_status_text(cooked_text) + if args.require_validation and not cooked_validation: + raise ValueError("cooked capture did not enable Khronos Vulkan validation") + + reference, candidate = read_ppm(source_image), read_ppm(cooked_image) + width, height, pixels = reference + background = pixels[:3] + different = sum(max(abs(r - background[0]), abs(g - background[1]), abs(b - background[2])) > 4 + for r, g, b in zip(pixels[0::3], pixels[1::3], pixels[2::3])) + if different * 500 < width * height: + raise RuntimeError("source capture is missing the fixture triangle") + if args.blend: + blended = sum(8 < r < background[0] - 8 and 8 < g < background[1] - 8 and 8 < b < background[2] - 8 + for r, g, b in zip(pixels[0::3], pixels[1::3], pixels[2::3])) + if blended * 500 < width * height: + raise RuntimeError("transparent fixture has no visible intermediate-color triangle") + comparison = compare_region(reference, candidate, (0, 0, width, height)) + report = {"scope": "generated source-versus-cooked smoke; reference review required", "material": "blend" if args.blend else "opaque", "hierarchy": args.hierarchy, "standalone": args.standalone, + "width": width, "height": height, + "source_validation_enabled": source_validation, "cooked_validation_enabled": cooked_validation, + "viewport": comparison} + if args.flat_reference: + report["equivalent_flat"] = compare_region(read_ppm(args.flat_reference), reference, (0, 0, width, height)) + if args.opaque_reference: + report["transparency"] = compare_transparency(read_ppm(args.opaque_reference), candidate) + (output / "comparison.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2)) + return 0 if comparison["passed"] and all(report[name]["passed"] for name in ("equivalent_flat", "transparency") if name in report) else 1 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as error: + print(f"Cooked scene parity failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/tests/scene/cooked_scene_smoke.py b/tests/scene/cooked_scene_smoke.py new file mode 100644 index 000000000..3154efd9a --- /dev/null +++ b/tests/scene/cooked_scene_smoke.py @@ -0,0 +1,739 @@ +#!/usr/bin/env python3 +"""Run the asset-bearing cooked scene in a fresh desktop Vulkan process. + +Requires built SceneCookProcess and Obelisk binaries and a usable desktop GPU. +This is a startup/lifetime smoke test, not a pixel or performance qualification. +""" +import argparse +import hashlib +import json +import pathlib +import platform +import re +import shutil +import subprocess +import sys +import tempfile +from validate_scene_lifecycle import distribution + + +SCENE = "10000000-0000-0000-0000-000000000001" +ALTERNATE_SCENE = "90000000-0000-0000-0000-000000000009" +MISSING_SCENE = "90000000-0000-0000-0000-000000000010" + + +def sha256_file(path): + with pathlib.Path(path).open("rb") as source: + return hashlib.file_digest(source, "sha256").hexdigest() + + +def read_process_logs(directory): + """Read spdlog's rotated files oldest first and reject possible data loss.""" + directory = pathlib.Path(directory) + rotated = {int(path.name.removeprefix("engine_dump.").removesuffix(".log")): path + for path in directory.glob("engine_dump.*.log") + if path.name.removeprefix("engine_dump.").removesuffix(".log").isdigit()} + if any(index < 1 or index >= 5 for index in rotated): + raise ValueError("process log reached its rotation limit; early evidence may be lost") + if rotated and sorted(rotated) != list(range(1, max(rotated) + 1)): + raise ValueError("process log has a missing rotated segment") + paths = [rotated[index] for index in sorted(rotated, reverse=True)] + [directory / "engine_dump.log"] + return "".join(path.read_text(encoding="utf-8") for path in paths) + + +def read_console_log(path): + with pathlib.Path(path).open("rb") as source: + data = source.read(16 * 1024 * 1024 + 1) + if len(data) > 16 * 1024 * 1024: + raise ValueError("process console output exceeds its evidence limit") + return data.decode("utf-8", errors="replace") + + +def require_clean_vulkan_console(text): + for line in text.splitlines(): + lowered = line.lower() + if any(marker in lowered for marker in ("vuid-", "validation error", "vk_error_validation_failed", "vk_not_ready")): + raise ValueError(f"Vulkan console validation failure: {line[:500]}") + + +def require_zero_source_audit(log): + """Require one complete shutdown audit, with every counter exactly zero.""" + lines = [line for line in log.splitlines() if "[CookedSourceAudit]" in line] + if len(lines) != 1: + raise ValueError("missing or repeated cooked source-access audit") + match = re.search(r"\[CookedSourceAudit\] vfs_denied=(\d+) registry_denied=(\d+) imports_denied=(\d+)(?:\s|$)", lines[0]) + if not match or tuple(map(int, match.groups())) != (0, 0, 0): + raise ValueError("cooked source-access audit is malformed or reports denied lookups") + + +def validate_standalone_package_failure(code, log): + """Require the package failure path and clean shutdown before publication.""" + markers = ("[CookedRuntime] cannot admit cooked scene request", "[CookedRuntime] cooked reconstruction failed") + if code != 2 or "Engine destroyed" not in log or "[CookedRuntime] Published cooked scene generation" in log or not any(marker in log for marker in markers): + raise RuntimeError("standalone package failure did not shut down before publication (exit {}):\n{}".format(code, "\n".join(log.splitlines()[-30:]))) + + +def validation_layer_status_text(log): + lines = [line for line in log.splitlines() if "[VulkanValidation]" in line] + if len(lines) != 1: + raise ValueError("missing or repeated Vulkan validation-layer status") + match = re.search(r"\[VulkanValidation\] khronos_enabled=([01])(?:\s|$)", lines[0]) + if not match: + raise ValueError("malformed Vulkan validation-layer status") + return match.group(1) == "1" + + +def validate_retry_publications(log): + """The failure probe retries the original scene after preserving it.""" + publications = [(int(generation), scene) for generation, scene in re.findall( + r"\[CookedRuntime\] Published cooked scene generation (\d+) scene=([0-9a-f-]+)", log)] + if publications != [(2, SCENE), (3, SCENE)]: + raise ValueError("failure retry published an unexpected scene or generation") + + +def read_cancellation_timings(log, cycles): + matches = re.findall(r"\[CookedCancelTiming\] cancelled=(\d+) boundary=(cpu|gpu) request_ns=(\d+) cleanup_ns=(\d+) elapsed_ns=(\d+)", log) + expected = (cycles + 1) // 2 + if len(matches) != expected: + raise ValueError("missing or repeated cancellation timing observations") + samples = [] + for index, (ordinal, boundary, requested, cleanup, elapsed) in enumerate(matches, 1): + requested, cleanup, elapsed = map(int, (requested, cleanup, elapsed)) + if int(ordinal) != index or boundary != ("cpu" if index % 2 else "gpu") or requested <= 0 or cleanup < requested or elapsed != cleanup - requested: + raise ValueError("invalid cancellation timing observation") + samples.append({"cancelled": index, "boundary": boundary, "request_ns": requested, "cleanup_ns": cleanup, "elapsed_ns": elapsed}) + return samples + + +def read_gpu_cancellation_handoffs(log, cancellations): + """Correlate optional profiling markers with complete GPU cancellations.""" + rows = re.findall(r"\[CookedGpuCancelTiming\] request_ns=(\d+) observed_ns=(\d+) finished_ns=(\d+)", log) + marker_count = log.count("[CookedGpuCancelTiming]") + if not marker_count: + return [] + gpu = [sample for sample in cancellations if sample["boundary"] == "gpu"] + if len(rows) != marker_count or len(rows) != len(gpu): + raise ValueError("missing or malformed GPU cancellation handoff") + result = [] + for sample, row in zip(gpu, rows): + requested, observed, finished = map(int, row) + if not sample["request_ns"] <= requested <= observed <= finished <= sample["cleanup_ns"]: + raise ValueError("GPU cancellation handoff is outside its cleanup interval") + result.append({"cancelled": sample["cancelled"], "request_to_observed_ns": observed - requested, + "observed_to_finished_ns": finished - observed, "finished_to_cleanup_ns": sample["cleanup_ns"] - finished}) + return result + + +def validate_stress_sequence(log, cycles): + """Check exact scene identities and post-cancellation generations.""" + publications = [(int(generation), scene) for generation, scene in re.findall( + r"\[CookedRuntime\] Published cooked scene generation (\d+) scene=([0-9a-f-]+)", log)] + committed = cycles // 2 + expected = [(generation, SCENE if generation % 2 == 0 else ALTERNATE_SCENE) + for generation in range(2, committed + 3)] + if publications != expected: + raise ValueError("stress published an unexpected scene or generation") + frames = [tuple(map(int, match)) for match in re.findall( + r"\[CookedStress\] cancelled=(\d+) generation=(\d+) rendered_after_cleanup=1", log)] + cancelled = (cycles + 1) // 2 + if frames != [(index, index + 1) for index in range(1, cancelled + 1)]: + raise ValueError("stress cancellation did not retain the expected scene generation") + + +def require_stable_stress_window(log): + """A resized render target invalidates a same-ledger comparison.""" + baseline = log.find("[CookedTiming] generation=2 ") + if baseline < 0: + raise ValueError("stress run has no initial timing baseline") + if "Window size updated" in log[baseline:] or "Swapchain recreated:" in log[baseline:]: + raise ValueError("stress viewport changed after the initial baseline; allocation comparisons require a fixed viewport") + dimensions = re.findall(r"\[ZUI\] UIScale=[^ ]+ Screen=(\d+)x(\d+) Window=(\d+)x(\d+) Framebuffer=(\d+)x(\d+)", log[:baseline]) + if not dimensions: + raise ValueError("stress run did not record initial window and framebuffer dimensions") + screen_w, screen_h, window_w, window_h, fb_w, fb_h = map(int, dimensions[-1]) + if (window_w, window_h) != (1280, 720) or min(screen_w, screen_h, fb_w, fb_h) <= 0: + raise ValueError("stress run did not use the configured 1280x720 window") + return {"scene_viewport": [screen_w, screen_h], "window": [window_w, window_h], "framebuffer": [fb_w, fb_h]} + + +def require_clean_stress_log(log): + if re.search(r"\[(?:error|critical)\]", log, re.IGNORECASE): + raise ValueError("stress run contains an engine error or critical log") + require_no_environment_fallback(log) + + +def require_no_environment_fallback(log): + if re.search(r"\[SkyEnvironment\].*using the fallback", log, re.IGNORECASE): + raise ValueError("cooked run used an environment fallback") + + +def observed_timing_viewport(log): + baseline = log.find("[CookedTiming] generation=2 ") + if baseline < 0: + return None + records = re.findall(r"\[ZUI\] UIScale=[^ ]+ Screen=(\d+)x(\d+) Window=(\d+)x(\d+) Framebuffer=(\d+)x(\d+)", log[:baseline]) + if not records: + return None + screen_w, screen_h, window_w, window_h, fb_w, fb_h = map(int, records[-1]) + after = log[baseline:] + return {"scene_viewport": [screen_w, screen_h], "window": [window_w, window_h], "framebuffer": [fb_w, fb_h], + "resize_after_baseline": "Window size updated" in after or "Swapchain recreated:" in after} + + +def read_lifecycle_observations(log, generations, require_gpu_completion=False, require_cache=True, require_gpu_budget=False): + pattern = (r"\[CookedLifecycle\] generation=(\d+) cpu_live=(\d+) cpu_peak=(\d+) staging_live=(\d+) staging_peak=(\d+) " + r"metadata_live=(\d+) metadata_blocks=(\d+) gpu_live=(\d+) gpu_peak=(\d+) " + r"retirement_queued=(\d+) retirement_draining=(\d+)") + fields = ("generation", "cpu_live", "cpu_peak", "staging_live", "staging_peak", "metadata_live", "metadata_blocks", + "gpu_live", "gpu_peak", "retirement_queued", "retirement_draining") + samples = [dict(zip(fields, map(int, match))) for match in re.findall(pattern, log)] + if [sample["generation"] for sample in samples] != generations: + raise RuntimeError("missing cooked lifecycle memory observations") + caches = [tuple(map(int, match)) for match in re.findall( + r"\[CookedCache\] generation=(\d+) closures=(\d+) leases=(\d+) retiring=(\d+) asset_versions=(\d+)", log)] + if (require_cache or caches) and [cache[0] for cache in caches] != generations: + raise RuntimeError("missing cooked closure cache observations") + for sample, (_, closures, leases, retiring, asset_versions) in zip(samples, caches): + sample.update(cache_closures=closures, cache_leases=leases, cache_retiring=retiring, cache_asset_versions=asset_versions) + gpu_budgets = [tuple(map(int, match)) for match in re.findall( + r"\[CookedGpuBudget\] generation=(\d+) admitted_peak=(\d+) geometry_peak=(\d+) texture_peak=(\d+) target_peak=(\d+) uniform_peak=(\d+) staging_peak=(\d+) readback_peak=(\d+)", log)] + if (require_gpu_budget or gpu_budgets) and [item[0] for item in gpu_budgets] != generations: + raise RuntimeError("missing or reordered GPU budget observations") + gpu_fields = ("gpu_admitted_peak", "gpu_geometry_peak", "gpu_texture_peak", "gpu_target_peak", "gpu_uniform_peak", "gpu_staging_peak", "gpu_readback_peak") + for sample, record in zip(samples, gpu_budgets): + values = record[1:] + limits = (4 << 30, 4 << 30, 4 << 30, 4 << 30, 4 << 30, 512 << 20, 256 << 20) + if any(value > limit for value, limit in zip(values, limits)) or any(value > values[0] for value in values[1:]): + raise RuntimeError("GPU admitted peak exceeds configured bounds") + sample.update(zip(gpu_fields, values)) + retirement_budgets = [tuple(map(int, match)) for match in re.findall( + r"\[CookedRetirementBudget\] generation=(\d+) capacity=(\d+) high_water=(\d+)", log)] + if (require_gpu_budget or retirement_budgets) and [item[0] for item in retirement_budgets] != generations: + raise RuntimeError("missing or reordered retirement budget observations") + for sample, (_, capacity, high_water) in zip(samples, retirement_budgets): + if not capacity or high_water > capacity: + raise RuntimeError("retirement high-water exceeds configured capacity") + sample.update(retirement_capacity=capacity, retirement_high_water=high_water) + timings = [tuple(map(int, match)) for match in re.findall( + r"\[CookedTiming\] generation=(\d+) admission_ns=(\d+) first_submission_ns=(\d+) elapsed_ns=(\d+)", log)] + if [timing[0] for timing in timings] != generations: + raise RuntimeError("missing cooked admission-to-submission timing observations") + for sample, (_, admission, submission, elapsed) in zip(samples, timings): + if not admission or submission < admission or elapsed != submission - admission: + raise RuntimeError("invalid cooked admission-to-submission timing") + sample.update(admission_ns=admission, first_submission_ns=submission, admission_to_submission_ns=elapsed) + completions = [tuple(map(int, match)) for match in re.findall( + r"\[CookedGpuCompletion\] generation=(\d+) observed_ns=(\d+) timeline_value=(\d+)", log)] + if require_gpu_completion and [value[0] for value in completions] != generations: + raise RuntimeError("missing render-completion observations") + if completions: + if [value[0] for value in completions] != generations: + raise RuntimeError("partial or reordered render-completion observations") + for sample, (_, observed, timeline) in zip(samples, completions): + if observed < sample["first_submission_ns"] or not timeline: + raise RuntimeError("invalid render-completion observation") + sample.update(render_completion_observed_ns=observed, render_timeline_value=timeline, + admission_to_completion_observed_ns=observed - sample["admission_ns"]) + if any(sample["cpu_live"] > 1024 * 1024 * 1024 or sample["cpu_peak"] > 1024 * 1024 * 1024 or sample["staging_live"] > 256 * 1024 * 1024 or sample["staging_peak"] > 256 * 1024 * 1024 or sample["metadata_live"] > 16 * 1024 * 1024 or sample["gpu_live"] > sample["gpu_peak"] for sample in samples): + raise RuntimeError("cooked lifecycle memory observations exceed their configured bounds") + return samples + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cooker", required=True, type=pathlib.Path) + parser.add_argument("--obelisk", required=True, type=pathlib.Path) + parser.add_argument("--timeout", type=float, default=120) + parser.add_argument("--failure-cases", action="store_true", help="also reject missing, corrupt and incompatible package objects") + parser.add_argument("--capture", type=pathlib.Path, help="write a scene viewport RGB8 PPM after rendering settles") + parser.add_argument("--verify-capture", action="store_true", help="capture and validate a temporary standalone fixture image") + parser.add_argument("--reloads", type=int, default=0, help="alternate two cooked scenes this many times") + parser.add_argument("--fresh-processes", type=int, default=1, help="run this many fresh processes against one prepared package") + parser.add_argument("--include-alternate", action="store_true", help="include a second cooked scene without requesting a replacement") + parser.add_argument("--runtime-camera", help="use the fixture's authored camera: 'main' or an explicit UUID") + parser.add_argument("--standalone", action="store_true", help="run the authored-camera fixture in the renderer-only host") + parser.add_argument("--audit-source", action="store_true", help="deny and count source VFS, registry, and import lookups") + parser.add_argument("--observe-gpu-completion", action="store_true", help="observe completion of the first accepted render submission") + parser.add_argument("--require-validation", action="store_true", help="require the Khronos Vulkan validation layer to be enabled") + parser.add_argument("--lifecycle-stress", action="store_true", help="alternate CPU and recorded-upload cancellations with committed scene switches") + parser.add_argument("--fixed-window", action="store_true", help="use a non-resizable 1280x720 logical test window") + parser.add_argument("--reject-runtime-camera", action="store_true", help="expect the selected runtime camera to be rejected") + faults = parser.add_mutually_exclusive_group() + faults.add_argument("--cancel-during-upload", action="store_true", help="cancel the first replacement after recording its mesh upload") + faults.add_argument("--cancel-during-cpu", action="store_true", help="cancel the first replacement while the CPU worker owns its reconstruction") + faults.add_argument("--fail-preflight", action="store_true", help="refuse publication after replacement resources are ready") + faults.add_argument("--fail-staging", action="store_true", help="refuse the replacement's GPU staging allocation") + faults.add_argument("--reject-submit", action="store_true", help="reject the replacement's recorded GPU batch before submission") + faults.add_argument("--missing-replacement", action="store_true", help="request a scene absent from the cooked package after the first scene renders") + faults.add_argument("--corrupt-replacement", action="store_true", help="corrupt only the replacement scene object before launch") + faults.add_argument("--missing-replacement-material", action="store_true", help="remove a material required only by the replacement") + faults.add_argument("--corrupt-replacement-material", action="store_true", help="corrupt a material required only by the replacement") + faults.add_argument("--missing-replacement-camera", action="store_true", help="omit the authored camera from the replacement scene") + faults.add_argument("--device-loss", action="store_true", help="inject device loss during a standalone replacement upload") + faults.add_argument("--shutdown-cpu", action="store_true", help="close while a replacement worker is active") + faults.add_argument("--shutdown-gpu", action="store_true", help="close with a recorded replacement upload") + parser.add_argument("--retry-after-failure", action="store_true", help="retry the original scene after a replacement fault") + parser.add_argument("--supersede-reload", action="store_true", help="supersede the first waiting replacement with a newer scene") + parser.add_argument("--supersede-during-cpu", action="store_true", help="supersede a replacement while its CPU worker holds resources") + parser.add_argument("--supersede-during-upload", action="store_true", help="supersede a replacement after recording its mesh upload") + parser.add_argument("--evidence", type=pathlib.Path, help="write per-generation memory and submission timing observations as new JSON") + parser.add_argument("--cache-conditions", default="uncontrolled; fresh process does not imply cold I/O", help="describe filesystem cache conditions; does not clear caches") + parser.add_argument("--stage-trace", type=pathlib.Path, help="write a bounded SceneLoad stage CSV from the cooked process") + parser.add_argument("--steady-frames", type=int, default=0, help="observe settled standalone frames before replacement") + args = parser.parse_args() + cooker, obelisk = args.cooker.resolve(strict=True), args.obelisk.resolve(strict=True) + binary_hashes = {"cooker_sha256": sha256_file(cooker), "runtime_sha256": sha256_file(obelisk)} + if args.timeout <= 0 or not 0 <= args.reloads <= (1100 if args.lifecycle_stress else 1000) or not 1 <= args.fresh_processes <= 1000: + parser.error("--timeout must be positive; --reloads and --fresh-processes must be within their bounds") + if not args.cache_conditions.strip(): + parser.error("--cache-conditions must describe the observed cache state") + if args.lifecycle_stress and (not args.standalone or not args.reloads or args.runtime_camera is None or args.retry_after_failure or args.cancel_during_upload or args.cancel_during_cpu or args.fail_staging or args.fail_preflight or args.reject_submit or args.missing_replacement or args.corrupt_replacement or args.corrupt_replacement_material or args.missing_replacement_material or args.missing_replacement_camera or args.device_loss or args.supersede_reload or args.supersede_during_cpu or args.supersede_during_upload or args.capture or args.steady_frames): + parser.error("--lifecycle-stress requires standalone replacements without other probes") + if args.audit_source and not args.standalone: + parser.error("--audit-source requires --standalone") + if args.fixed_window and not args.standalone: + parser.error("--fixed-window requires --standalone") + if args.observe_gpu_completion and (not args.standalone or args.device_loss or args.shutdown_cpu or args.shutdown_gpu): + parser.error("--observe-gpu-completion requires standalone non-fatal execution") + if args.include_alternate and (not args.standalone or args.runtime_camera is None or args.reloads or args.failure_cases): + parser.error("--include-alternate requires a standalone authored-camera startup without replacements or package-failure tests") + if (args.shutdown_cpu or args.shutdown_gpu) and (not args.standalone or args.reloads != 1 or args.retry_after_failure or args.lifecycle_stress or args.failure_cases or args.capture or args.steady_frames or args.supersede_reload or args.supersede_during_cpu or args.supersede_during_upload): + parser.error("shutdown probes require one standalone replacement without other probes") + if args.steady_frames < 0 or args.steady_frames > 600 or (args.steady_frames and (not args.standalone or args.fresh_processes != 1 or args.capture or args.verify_capture or args.failure_cases or args.retry_after_failure or args.supersede_reload or args.supersede_during_cpu or args.supersede_during_upload or args.reject_runtime_camera)): + parser.error("--steady-frames requires a standalone run without capture or fault probes") + if args.evidence is not None and args.evidence.exists(): + parser.error("--evidence must name a new file") + evidence_log = args.evidence.with_suffix(args.evidence.suffix + ".engine.log") if args.evidence else None + if evidence_log is not None and (evidence_log.exists() or (args.fresh_processes > 1 and any(evidence_log.parent.glob(evidence_log.name + ".process-*.log")))): + parser.error("the evidence log path must not already exist") + if args.stage_trace is not None and (args.stage_trace.exists() or args.failure_cases): + parser.error("--stage-trace must name a new file and cannot be combined with --failure-cases") + if (args.cancel_during_upload or args.cancel_during_cpu or args.fail_staging or args.fail_preflight or args.reject_submit or args.missing_replacement or args.corrupt_replacement or args.corrupt_replacement_material or args.missing_replacement_material or args.missing_replacement_camera or args.device_loss) and (args.reloads != 1 or args.failure_cases or args.capture): + parser.error("replacement fault probes require --reloads 1 and no capture or failure cases") + if args.retry_after_failure and not (args.corrupt_replacement_material or args.missing_replacement_material or args.missing_replacement_camera or args.missing_replacement or args.cancel_during_cpu or args.corrupt_replacement or args.cancel_during_upload or args.fail_staging or args.fail_preflight or args.reject_submit): + parser.error("--retry-after-failure requires a replacement fault probe") + if (args.corrupt_replacement_material or args.missing_replacement_material) and (not args.standalone or args.runtime_camera != "main" or not args.retry_after_failure): + parser.error("replacement material faults require standalone main camera and retry") + if args.missing_replacement_camera and (not args.standalone or args.runtime_camera != "main" or not args.retry_after_failure): + parser.error("--missing-replacement-camera requires standalone main camera and retry") + if args.fail_preflight and (not args.standalone or not args.retry_after_failure): + parser.error("--fail-preflight requires standalone and retry") + if args.reject_submit and not args.retry_after_failure: + parser.error("--reject-submit requires --retry-after-failure") + if args.device_loss and (not args.standalone or args.retry_after_failure or args.steady_frames or args.stage_trace): + parser.error("--device-loss requires a standalone replacement without retry or timing probes") + if args.fresh_processes > 1 and (args.reloads or args.failure_cases or args.capture or args.stage_trace or args.retry_after_failure): + parser.error("multiple fresh processes cannot be combined with replacements, faults, capture, or stage tracing") + if (args.supersede_reload or args.supersede_during_cpu or args.supersede_during_upload) and (args.reloads != 1 or sum((args.supersede_reload, args.supersede_during_cpu, args.supersede_during_upload)) > 1 or args.failure_cases or args.capture or args.cancel_during_upload or args.cancel_during_cpu or args.fail_staging or args.fail_preflight or args.reject_submit or args.missing_replacement or args.corrupt_replacement or args.corrupt_replacement_material or args.missing_replacement_material or args.missing_replacement_camera or args.retry_after_failure): + parser.error("supersession probes require one replacement and no other fault or capture") + if args.runtime_camera is not None and (not args.runtime_camera or (args.reloads and not args.standalone) or (args.failure_cases and not args.standalone) or (args.fresh_processes != 1 and not args.standalone)): + parser.error("runtime-camera fixture requires one fresh process without replacements or failure cases") + if args.reject_runtime_camera and (args.runtime_camera in (None, "main") or args.capture or args.stage_trace): + parser.error("--reject-runtime-camera requires an explicit UUID without capture or stage tracing") + if args.standalone and args.runtime_camera is None: + parser.error("--standalone requires an authored camera") + if args.verify_capture and (not args.standalone or args.capture or args.reloads or args.reject_runtime_camera or args.failure_cases): + parser.error("--verify-capture requires one standalone authored-camera startup") + if args.standalone and ((args.retry_after_failure and not (args.corrupt_replacement_material or args.missing_replacement_material or args.missing_replacement_camera or args.missing_replacement or args.corrupt_replacement or args.cancel_during_upload or args.cancel_during_cpu or args.reject_submit or args.fail_staging or args.fail_preflight)) or ((args.missing_replacement or args.corrupt_replacement or args.cancel_during_upload or args.cancel_during_cpu or args.fail_staging) and not args.retry_after_failure)): + parser.error("unsupported standalone fault probe") + + with tempfile.TemporaryDirectory(prefix="renderer-cooked-smoke-") as temporary: + root = pathlib.Path(temporary) + authoring = root / "authoring" + authoring.mkdir() + mode = "write-switch-unique-material" if (args.corrupt_replacement_material or args.missing_replacement_material) else "write-switch-no-camera" if args.missing_replacement_camera else "write-switch-camera" if args.runtime_camera is not None and (args.reloads or args.include_alternate) else "write-camera" if args.runtime_camera is not None else "write-switch" if args.reloads else "write" + subprocess.run([str(cooker), mode, str(authoring)], check=True, timeout=args.timeout) + if sha256_file(cooker) != binary_hashes["cooker_sha256"]: + raise RuntimeError("cooker executable changed while creating the fixture") + package = root / "package" + shutil.copytree(authoring / "output" / "desktop-vulkan-le64", package) + with (package / "manifest.json").open(encoding="utf-8") as source: + manifest = json.load(source) + manifest_sha256 = hashlib.sha256((package / "manifest.json").read_bytes()).hexdigest() + if len(manifest.get("assets", [])) != (5 if (args.corrupt_replacement_material or args.missing_replacement_material) else 4): + raise RuntimeError("fixture has the wrong number of cooked assets") + if len(manifest.get("scenes", [])) != (2 if (args.reloads or args.include_alternate) else 1): + raise RuntimeError("fixture has the wrong number of cooked scenes") + if args.corrupt_replacement_material or args.missing_replacement_material: + asset = next(asset for asset in manifest["assets"] if asset["uuid"] == "b0000000-0000-0000-0000-00000000000b") + object_path = package / asset["object"]["path"] + if args.missing_replacement_material: + object_path.unlink() + else: + data = bytearray(object_path.read_bytes()) + data[-1] ^= 1 + object_path.write_bytes(data) + if args.corrupt_replacement: + scene = next(scene for scene in manifest["scenes"] if scene["uuid"] == ALTERNATE_SCENE) + object_path = package / scene["object"]["path"] + data = bytearray(object_path.read_bytes()) + data[-1] ^= 1 + object_path.write_bytes(data) + shutil.rmtree(authoring) + run_count = 0 + observations = [] + validation_enabled = False + validation_states = [] + timing_viewports = [] + stress_window = None + cancellation_timings = [] + cancellation_handoffs = [] + + def run(candidate: pathlib.Path, expected_draws: int = 1, capture=None, expected_startup_failure=False) -> int: + nonlocal run_count, validation_enabled, stress_window, cancellation_timings, cancellation_handoffs + run_count += 1 + log_directory = root / f"log-{run_count}" + log_directory.mkdir() + command = [ + str(obelisk), *( ["--cookedStandalone"] if args.standalone else ["--launchEditor", "true"] ), "--cookedPackage", str(candidate), + "--cookedScene", SCENE, "--cookedExitAfterFrame", "--cookedExpectedDraws", str(expected_draws), + "--logDirectory", str(log_directory), + ] + if capture is not None: + command.extend(["--cookedCapture", str(capture.resolve())]) + if args.runtime_camera is not None: + if not args.standalone: + command.append("--cookedRuntimeView") + if args.runtime_camera != "main": + command.extend(["--cookedCamera", args.runtime_camera]) + if args.audit_source: + command.append("--cookedAuditSource") + if args.observe_gpu_completion: + command.append("--cookedObserveGpuCompletion") + if args.lifecycle_stress: + command.append("--cookedLifecycleStress") + if args.fixed_window: + command.append("--cookedFixedWindow") + if args.reloads: + command.extend(["--cookedReloads", str(args.reloads), "--cookedAlternateScene", MISSING_SCENE if args.missing_replacement else ALTERNATE_SCENE]) + if args.cancel_during_upload: + command.append("--cookedCancelReload") + if args.cancel_during_cpu: + command.append("--cookedCancelCpu") + if args.fail_preflight: + command.append("--cookedFailPreflight") + if args.fail_staging: + command.append("--cookedFailReloadStaging") + if args.reject_submit: + command.append("--cookedRejectReloadSubmit") + if args.device_loss: + command.append("--cookedInjectDeviceLoss") + if args.shutdown_cpu: + command.append("--cookedShutdownCpu") + if args.shutdown_gpu: + command.append("--cookedShutdownGpu") + if args.retry_after_failure: + command.append("--cookedRetryFailure") + if args.supersede_reload: + command.append("--cookedSupersedeReload") + if args.supersede_during_cpu: + command.append("--cookedSupersedeCpu") + if args.supersede_during_upload: + command.append("--cookedSupersedeGpu") + if args.stage_trace is not None: + command.extend(["--cookedStageTrace", str(args.stage_trace.resolve())]) + if args.steady_frames: + command.extend(["--cookedSteadyFrames", str(args.steady_frames)]) + stdout_path, stderr_path = log_directory / "stdout.log", log_directory / "stderr.log" + try: + with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + code = subprocess.run(command, timeout=args.timeout, stdout=stdout, stderr=stderr).returncode + finally: + if evidence_log is not None and (log_directory / "engine_dump.log").exists(): + # Save even when the process times out or later assertions fail. + destination = evidence_log.with_name(evidence_log.name + f".process-{run_count:04d}.log") if args.fresh_processes > 1 or args.failure_cases else evidence_log + destination.write_text(read_process_logs(log_directory), encoding="utf-8") + for stream in (stdout_path, stderr_path): + if stream.exists(): + destination.with_name(destination.name + "." + stream.name).write_text(read_console_log(stream), encoding="utf-8") + log = read_process_logs(log_directory) + require_clean_vulkan_console(read_console_log(stdout_path)) + require_clean_vulkan_console(read_console_log(stderr_path)) + if sha256_file(obelisk) != binary_hashes["runtime_sha256"]: + raise RuntimeError("runtime executable changed during the observation") + validation_enabled = validation_layer_status_text(log) + validation_states.append(validation_enabled) + timing_viewports.append(observed_timing_viewport(log)) + if args.require_validation and not validation_enabled: + raise RuntimeError("Khronos Vulkan validation layer was not enabled") + if any(marker in log.lower() for marker in ("vuid-", "validation error", "vk_not_ready")): + raise RuntimeError("Vulkan validation error during cooked lifecycle smoke") + if args.standalone: + if args.audit_source: + require_zero_source_audit(log) + if expected_startup_failure: + validate_standalone_package_failure(code, log) + return 1 + if args.failure_cases and expected_draws != 1: + if code != 2 or "submitted draw count does not match fixture" not in log or "Engine destroyed" not in log: + raise RuntimeError("standalone draw-count mismatch was not rejected cleanly") + return 2 + if args.reloads: + require_no_environment_fallback(log) + if args.device_loss: + if code != 2 or "Injecting device loss after upload recording" not in log or "Device lost; entering fatal shutdown" not in log or "Published cooked scene generation 3" in log or "Active scene submitted after failure" in log or "Engine destroyed" not in log: + raise RuntimeError("standalone runtime did not enter fatal device-loss shutdown (exit {}):\n{}".format(code, "\n".join(log.splitlines()[-30:]))) + return code + if args.shutdown_cpu or args.shutdown_gpu: + boundary = "CPU worker pending" if args.shutdown_cpu else "GPU upload pending" + if code != 0 or f"Shutdown requested with {boundary}" not in log or "Shutdown cancelled and drained pending replacement" not in log or "Published cooked scene generation 3" in log or "Engine destroyed" not in log: + raise RuntimeError("standalone shutdown did not cancel and drain its replacement (exit {}):\n{}".format(code, "\n".join(log.splitlines()[-30:]))) + return code + if args.reject_runtime_camera: + if code != 2 or "requested runtime camera is missing, ambiguous, or has no transform" not in log or "Published cooked scene generation" in log or "Engine destroyed" not in log: + raise RuntimeError("standalone runtime did not reject the missing camera before publication (exit {}):\n{}".format(code, "\n".join(log.splitlines()[-30:]))) + return 1 + generations = [int(value) for value in re.findall(r"\[CookedRuntime\] Published cooked scene generation (\d+)", log)] + if args.supersede_reload and ("Superseded waiting request before worker admission" not in log or "Committed request outcome retained and acknowledged" not in log or f"Published cooked scene generation 3 scene={SCENE}" not in log): + raise RuntimeError("standalone runtime did not supersede the waiting request and publish the latest scene") + if args.cancel_during_upload and ("Cancelled replacement after mesh upload recording" not in log or "Cancelled replacement drained; active scene generation 2 retained" not in log): + raise RuntimeError("standalone runtime did not cancel and drain the recorded upload") + if args.cancel_during_cpu and ("Cancelled replacement while CPU worker owned its inputs" not in log or "Cancelled worker outcome acknowledged before cleanup" not in log or "Cancelled replacement drained; active scene generation 2 retained" not in log): + raise RuntimeError("standalone runtime did not cancel and drain the CPU worker") + if args.supersede_during_cpu and ("Superseded CPU worker with cleanup pending" not in log or "Superseded worker outcome acknowledged before cleanup" not in log or "Superseded replacement drained; active scene retained" not in log): + raise RuntimeError("standalone runtime did not supersede and drain the CPU worker") + if args.supersede_during_upload and ("Superseded recorded upload with cleanup pending" not in log or "Superseded replacement drained; active scene retained" not in log): + raise RuntimeError("standalone runtime did not supersede and drain the recorded upload") + if args.reject_submit and log.count("[CookedUploadProbe] Rejected recorded cooked batch before queue submission") != 1: + raise RuntimeError("standalone runtime did not reject exactly one recorded upload batch") + if args.fail_preflight and "[CookedRuntime] injected publication preflight refusal" not in log: + raise RuntimeError("standalone runtime did not refuse the ready candidate before publication") + if args.fail_staging and "[CookedRuntime] Replacement staging allocation was refused" not in log: + raise RuntimeError("standalone replacement did not encounter a staging budget refusal") + if args.corrupt_replacement and "[CookedRuntime] cooked reconstruction failed" not in log: + raise RuntimeError("standalone runtime did not reject the corrupted scene object during reconstruction") + if (args.corrupt_replacement_material or args.missing_replacement_material) and "[CookedRuntime] Asset closure reconstruction failed" not in log: + raise RuntimeError("standalone runtime did not reject the invalid replacement material") + if args.missing_replacement_camera and "[CookedRuntime] runtime camera is invalid" not in log: + raise RuntimeError("standalone runtime did not reject the replacement without a camera") + if args.retry_after_failure and (((args.corrupt_replacement_material or args.missing_replacement_material or args.missing_replacement_camera or args.missing_replacement or args.corrupt_replacement or args.reject_submit or args.fail_staging or args.fail_preflight) and "Replacement failed; active scene generation 2 retained" not in log) or "Active scene submitted after failure; retry admitted" not in log): + raise RuntimeError("standalone runtime did not render its outgoing scene after failure and admit retry") + if args.retry_after_failure and (args.corrupt_replacement_material or args.missing_replacement_material or args.missing_replacement_camera or args.missing_replacement or args.corrupt_replacement or args.reject_submit or args.fail_staging or args.fail_preflight) and "Failed replacement outcome retained and acknowledged" not in log: + raise RuntimeError("standalone runtime did not retain and acknowledge the failed replacement outcome") + if args.retry_after_failure: + validate_retry_publications(log) + committed = args.reloads // 2 if args.lifecycle_stress else args.reloads + if args.lifecycle_stress: + cancelled = (args.reloads + 1) // 2 + cpu_cancelled = (args.reloads + 3) // 4 + gpu_cancelled = (args.reloads + 1) // 4 + validate_stress_sequence(log, args.reloads) + cancellation_timings = read_cancellation_timings(log, args.reloads) + cancellation_handoffs = read_gpu_cancellation_handoffs(log, cancellation_timings) + stress_window = require_stable_stress_window(log) + require_clean_stress_log(log) + if (f"[CookedStress] admitted={args.reloads} cancelled={cancelled} committed={committed}" not in log + or len(re.findall(r"\[CookedStress\] cancelled=\d+ generation=\d+ rendered_after_cleanup=1", log)) != cancelled + or log.count("Cancelled worker outcome acknowledged before cleanup") != cpu_cancelled + or log.count("Cancelled upload outcome acknowledged before cleanup") != gpu_cancelled): + raise RuntimeError("lifecycle stress did not observe every terminal outcome and post-cancellation frame") + if code != 0 or generations != list(range(2, committed + 3)) or "[CookedRuntime] Submitted 1 draws" not in log or "Engine destroyed" not in log: + raise RuntimeError("standalone cooked runtime did not publish and submit its fixture through normal shutdown (exit {}):\n{}".format(code, "\n".join(log.splitlines()[-30:]))) + observations[:] = read_lifecycle_observations(log, generations, args.observe_gpu_completion, require_gpu_budget=True) + if args.lifecycle_stress: + if any(sample["cache_closures"] or sample["cache_leases"] or sample["cache_retiring"] or sample["cache_asset_versions"] for sample in observations): + raise RuntimeError("unleased cooked closures were not reclaimed after publication") + reference = [sample for sample in observations if sample["generation"] >= 4 and sample["generation"] % 2 == 0] + if len(reference) > 1: + fields = ("cpu_live", "staging_live", "metadata_live", "metadata_blocks", "gpu_live", "retirement_queued", "retirement_draining") + baseline = tuple(reference[0][field] for field in fields) + if baseline[-2:] != (0, 0) or any(tuple(sample[field] for field in fields) != baseline for sample in reference[1:]): + raise RuntimeError("tracked allocations did not return to the warmed reference scene baseline") + if args.steady_frames: + windows = [tuple(map(int, match)) for match in re.findall(r"\[CookedSteady\] generation=(\d+) start_ns=(\d+) end_ns=(\d+) submitted_frames=(\d+)", log)] + if len(windows) != 1: + raise RuntimeError("missing or repeated standalone steady-rendering window") + generation, start, end, frames = windows[0] + if generation != generations[0] or frames < args.steady_frames or start < observations[0]["first_submission_ns"] or end <= start or (len(observations) > 1 and end > observations[1]["admission_ns"]): + raise RuntimeError("invalid standalone steady-rendering window") + observations[0].update(steady_start_ns=start, steady_end_ns=end, steady_submitted_frames=frames) + if capture is not None: + from compare_scene_capture import read_ppm, validate_capture_log_text + validate_capture_log_text(log) + width, height, pixels = read_ppm(capture) + if width < 2 or height < 2 or min(pixels) == max(pixels): + raise RuntimeError("standalone scene capture has no visible variation") + return code + if code == 0: + if args.retry_after_failure: + # This run deliberately logs the rejected request as an + # error. It is not suitable as pixel-capture evidence. + if "Engine destroyed" not in log: + raise RuntimeError("retry did not shut down normally") + else: + from compare_scene_capture import validate_capture_log_text + validate_capture_log_text(log) + generations = [int(value) for value in re.findall(r"Published cooked scene generation (\d+)", log)] + if generations != list(range(2, args.reloads + 3)): + raise RuntimeError(f"unexpected cooked publication sequence: {generations}") + observations[:] = read_lifecycle_observations(log, generations, require_cache=False) + elif code == 1 and ("Outgoing scene submitted after cooked startup failure" not in log or "Engine destroyed" not in log): + raise RuntimeError("failure did not preserve and render the outgoing scene through shutdown") + if args.cancel_during_upload and "Cancelled cooked replacement after mesh upload recording" not in log: + raise RuntimeError("replacement did not reach the recorded upload cancellation boundary") + if args.cancel_during_cpu and "Cancelled cooked replacement during CPU preparation with worker cleanup pending" not in log: + raise RuntimeError("replacement did not reach the CPU worker cancellation boundary") + if args.fail_staging and "Cooked replacement staging allocation was refused" not in log: + raise RuntimeError("replacement did not encounter a staging budget refusal") + if args.reject_submit and log.count("[CookedUploadProbe] Rejected recorded cooked batch before queue submission") != 1: + raise RuntimeError("replacement did not encounter exactly one recorded upload submission rejection") + expected_publications = ["2", "3"] if args.retry_after_failure else ["2"] + if args.missing_replacement and ("Cooked package capture failed" not in log or re.findall(r"Published cooked scene generation (\d+)", log) != expected_publications): + raise RuntimeError("missing replacement did not fail package capture after the first cooked scene was published") + if args.corrupt_replacement and ("Cooked scene reconstruction failed" not in log or re.findall(r"Published cooked scene generation (\d+)", log) != expected_publications): + raise RuntimeError("corrupt replacement did not fail reconstruction after the first cooked scene was published") + if args.retry_after_failure and ("Outgoing scene submitted after cooked startup failure" not in log or "Retrying cooked scene after preserved failure" not in log): + raise RuntimeError("retry did not follow an observed preserved failure") + if args.supersede_reload and "Superseded cooked waiting request with a newer scene" not in log: + raise RuntimeError("cooked waiting request was not superseded") + if args.supersede_during_cpu and "Superseded cooked CPU worker with cleanup pending" not in log: + raise RuntimeError("cooked CPU worker was not superseded with cleanup pending") + if args.supersede_during_upload and "Superseded cooked GPU upload with cleanup pending" not in log: + raise RuntimeError("cooked GPU upload was not superseded with cleanup pending") + if args.runtime_camera is not None and not args.reject_runtime_camera: + positions = re.findall(r"\[CookedRuntimeCamera\] (prepared|capture)_position=([^\s]+) ([^\s]+) ([^\s]+)", log) + observed = {kind: tuple(map(float, (x, y, z))) for kind, x, y, z in positions} + if observed.get("prepared") != (1.0, 2.0, 8.0) or (capture is not None and observed.get("capture") != (1.0, 2.0, 8.0)): + raise RuntimeError("authored runtime camera was not used in the prepared and captured frames") + if args.reject_runtime_camera and "requested runtime camera is missing, ambiguous, or has no transform" not in log: + raise RuntimeError("missing runtime camera did not fail with the expected diagnostic") + return code + + expected_status = 2 if args.device_loss else 1 if args.reject_runtime_camera or ((args.cancel_during_upload or args.cancel_during_cpu or args.fail_staging or args.missing_replacement or args.corrupt_replacement) and not args.retry_after_failure) else 0 + capture_path = root / "standalone-capture.ppm" if args.verify_capture else args.capture + if run(package, capture=capture_path) != expected_status: + raise RuntimeError("valid package did not submit its expected draw") + if args.device_loss: + print("Injected device loss after recording; runtime entered fatal shutdown without publishing the replacement.") + return 0 + if args.shutdown_cpu or args.shutdown_gpu: + print("Shutdown cancelled and drained the pending cooked replacement.") + return 0 + if args.reject_runtime_camera: + print("Rejected the missing runtime camera and preserved the outgoing scene.") + return 0 + fresh_observations = [observations[0]] if args.fresh_processes > 1 else [] + for process in range(1, args.fresh_processes): + if run(package) != 0 or len(observations) != 1: + raise RuntimeError(f"fresh process {process + 1} failed") + fresh_observations.append(observations[0]) + print(f"Completed fresh cooked process {process + 1}/{args.fresh_processes}.", flush=True) + if args.retry_after_failure: + if args.evidence is not None: + with args.evidence.open("x", encoding="utf-8") as target: + json.dump({"scope": "cooked replacement failure and retry smoke", "outgoing_scene_submitted_after_failure": True, + "retry_published_and_submitted": True, "normal_shutdown": True, "samples": observations, + "khronos_validation_enabled": validation_enabled}, target, indent=2) + target.write("\n") + print("Preserved the outgoing cooked scene after failure and successfully submitted a retry.") + return 0 + if args.cancel_during_upload: + if args.evidence is not None: + with args.evidence.open("x", encoding="utf-8") as target: + json.dump({"scope": "recorded cooked mesh upload cancellation smoke", "cancelled_after_recording": True, + "outgoing_scene_submitted": True, "normal_shutdown": True, "validation_error_markers_seen": False, "khronos_validation_enabled": validation_enabled}, target, indent=2) + target.write("\n") + print("Cancelled a recorded mesh upload; the outgoing scene rendered and shut down normally.") + return 0 + if args.cancel_during_cpu: + if args.evidence is not None: + with args.evidence.open("x", encoding="utf-8") as target: + json.dump({"scope": "cooked CPU preparation cancellation smoke", "cancelled_with_worker_cleanup_pending": True, + "outgoing_scene_submitted": True, "normal_shutdown": True, "validation_error_markers_seen": False, "khronos_validation_enabled": validation_enabled}, target, indent=2) + target.write("\n") + print("Cancelled a cooked CPU preparation; the outgoing scene rendered and shut down normally.") + return 0 + if args.missing_replacement: + if args.evidence is not None: + with args.evidence.open("x", encoding="utf-8") as target: + json.dump({"scope": "missing cooked replacement scene smoke", "initial_cooked_scene_published": True, + "replacement_package_capture_failed": True, "outgoing_scene_submitted": True, + "normal_shutdown": True, "validation_error_markers_seen": False, "khronos_validation_enabled": validation_enabled}, target, indent=2) + target.write("\n") + print("Rejected a missing cooked replacement; the outgoing cooked scene rendered and shut down normally.") + return 0 + if args.corrupt_replacement: + if args.evidence is not None: + with args.evidence.open("x", encoding="utf-8") as target: + json.dump({"scope": "corrupt cooked replacement scene smoke", "initial_cooked_scene_published": True, + "replacement_reconstruction_failed": True, "outgoing_scene_submitted": True, + "normal_shutdown": True, "validation_error_markers_seen": False, "khronos_validation_enabled": validation_enabled}, target, indent=2) + target.write("\n") + print("Rejected a corrupt cooked replacement; the outgoing cooked scene rendered and shut down normally.") + return 0 + if args.fail_staging: + if args.evidence is not None: + with args.evidence.open("x", encoding="utf-8") as target: + json.dump({"scope": "cooked GPU staging allocation refusal smoke", "staging_refused": True, + "outgoing_scene_submitted": True, "normal_shutdown": True, "validation_error_markers_seen": False, "khronos_validation_enabled": validation_enabled}, target, indent=2) + target.write("\n") + print("Refused cooked GPU staging; the outgoing scene rendered and shut down normally.") + return 0 + if args.capture is not None: + from compare_scene_capture import read_ppm + width, height, pixels = read_ppm(args.capture) + # This generated fixture has a red HDR cubemap and a dark triangle. + # The assertions are semantic smoke checks, not visual qualification. + red = dark = 0 + for r, g, b in zip(pixels[0::3], pixels[1::3], pixels[2::3]): + red += r > 180 and r > 2 * g and r > 2 * b + dark += r < 8 and g < 8 and b < 8 + if red * 2 < width * height or dark * 500 < width * height: + raise RuntimeError("capture is missing the fixture's red HDR background or triangle") + print(f"Captured {width}x{height} scene viewport to {args.capture}.") + print("Cooked asset fixture submitted eight frames with one draw and shut down normally.") + if args.reloads: + print(f"Completed {args.reloads} lifecycle cycles." if args.lifecycle_stress else f"Completed {args.reloads} alternating cooked scene replacements.") + if args.failure_cases: + if run(package, expected_draws=2) != 2: + raise RuntimeError("incorrect draw expectation was not rejected") + print("Rejected incorrect expected draw count.") + for case in ("missing", "corrupt", "target", "format"): + damaged = root / case + shutil.copytree(package, damaged) + ledger = json.loads((damaged / "manifest.json").read_text(encoding="utf-8")) + asset = damaged / ledger["assets"][0]["object"]["path"] + if case == "missing": + asset.unlink() + elif case == "corrupt": + data = bytearray(asset.read_bytes()) + data[-1] ^= 1 + asset.write_bytes(data) + else: + if case == "target": + ledger["target"] = "unsupported-target" + else: + ledger["assets"][0]["format"] = "unsupported-format" + (damaged / "manifest.json").write_text(json.dumps(ledger), encoding="utf-8") + if run(damaged, expected_startup_failure=args.standalone) != 1: + raise RuntimeError(f"{case} package did not fail startup cleanly") + print(f"Rejected {case} cooked package.") + if args.evidence is not None: + with args.evidence.open("x", encoding="utf-8") as target: + json.dump({"scope": "cooked lifecycle smoke; not a calibrated qualification profile", + "reloads": args.reloads, "samples": observations, + "lifecycle_stress": args.lifecycle_stress, + "cancellation_timings": cancellation_timings, + "gpu_cancellation_handoffs": cancellation_handoffs, + "cancellation_latency_ms": {boundary: distribution([sample["elapsed_ns"] / 1_000_000 for sample in cancellation_timings if sample["boundary"] == boundary]) for boundary in ("cpu", "gpu") if any(sample["boundary"] == boundary for sample in cancellation_timings)}, + "stress_window": stress_window, + "khronos_validation_enabled": validation_enabled, + "validation_states": validation_states, + "source_audit_enabled": args.audit_source, + "gpu_completion_observed": args.observe_gpu_completion, + "timing_viewports": timing_viewports, + "binary_sha256": binary_hashes, + "host_system": platform.platform(), "host_machine": platform.machine(), + "manifest_sha256": manifest_sha256, + "fresh_process_samples": fresh_observations, + "filesystem_cache_conditions": args.cache_conditions}, target, indent=2) + target.write("\n") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as error: + print(f"Cooked scene smoke failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/tests/scene/evidence/README.md b/tests/scene/evidence/README.md new file mode 100644 index 000000000..3b14fd49a --- /dev/null +++ b/tests/scene/evidence/README.md @@ -0,0 +1,18 @@ +# Scene runtime evidence + +These files retain the observations cited by the +[M2 profile candidate](../../../docs/design/future-plan/scene-runtime-m2-profile-candidate.md). Each archive +contains evidence from its named run; it does not imply that the current build +or a production profile is qualified. Consult the profile for runtime identity, +scope, results and limitations. + +- [capacity](capacity/) contains synthetic workload, draw-limit and texture + admission observations. +- [failures](failures/) contains package-failure observations. +- [lifecycle](lifecycle/) contains cancellation, replacement and soak evidence. +- [performance](performance/) contains timing traces and presentation diagnostics. +- [visual](visual/) contains SampleProject capture and semantic-control evidence. + +The reviewed reference images remain in `ZEngine/tests/Rendering/Fixtures`; +strict performance and calibrated-resource qualification is tracked in +[follow-up #880](https://github.com/JeanPhilippeKernel/RendererEngine/issues/880). diff --git a/tests/scene/evidence/capacity/capacity-cancellation-8192-20261117.json b/tests/scene/evidence/capacity/capacity-cancellation-8192-20261117.json new file mode 100644 index 000000000..79648abb9 --- /dev/null +++ b/tests/scene/evidence/capacity/capacity-cancellation-8192-20261117.json @@ -0,0 +1,125 @@ +{ + "scope": "synthetic shared-mesh cooked capacity observation; not a calibrated profile", + "draws": 8192, + "entities_including_camera": 8193, + "limit_plus_one_refused": false, + "active_scene_retained_and_retried": false, + "unique_replacement_material": false, + "scene_sha256": "cb8985faffff37e46e0a9f89d6014be2344d4dcf99d444c578d49b1c43c61f89", + "manifest_sha256": "98dfacc0ccc5f5a5219d8d97396489d1e16f5ee475488a01337d7b9f114a7ed0", + "binary_sha256": { + "cooker_sha256": "cc4bc8c155a47c8a021030132134589bd4f7d68fc74f1568e69ff22c4294ba1f", + "runtime_sha256": "3e0bf49dbdaf11196587a2d9ba6d1f3b5af15a8a5d25ba82010aebe860ad658a" + }, + "host_system": "macOS-26.5.2-arm64-arm-64bit-Mach-O", + "host_machine": "arm64", + "filesystem_cache_conditions": "uncontrolled", + "samples": [ + { + "generation": 2, + "cpu_live": 1114272, + "cpu_peak": 2818208, + "staging_live": 160, + "staging_peak": 32928, + "metadata_live": 144, + "metadata_blocks": 1, + "gpu_live": 1490615208, + "gpu_peak": 1517812648, + "retirement_queued": 0, + "retirement_draining": 0, + "cache_closures": 0, + "cache_leases": 0, + "cache_retiring": 0, + "cache_asset_versions": 0, + "gpu_admitted_peak": 1517815040, + "gpu_geometry_peak": 1076625408, + "gpu_texture_peak": 0, + "gpu_target_peak": 69561600, + "gpu_uniform_peak": 304519168, + "gpu_staging_peak": 69206016, + "gpu_readback_peak": 0, + "retirement_capacity": 2047, + "retirement_high_water": 265, + "admission_ns": 349716079804958, + "first_submission_ns": 349722540964333, + "admission_to_submission_ns": 6461159375 + }, + { + "generation": 3, + "cpu_live": 1114272, + "cpu_peak": 2818208, + "staging_live": 160, + "staging_peak": 32928, + "metadata_live": 144, + "metadata_blocks": 1, + "gpu_live": 1490615336, + "gpu_peak": 1532969812, + "retirement_queued": 0, + "retirement_draining": 0, + "cache_closures": 0, + "cache_leases": 0, + "cache_retiring": 0, + "cache_asset_versions": 0, + "gpu_admitted_peak": 1532973184, + "gpu_geometry_peak": 1077018624, + "gpu_texture_peak": 0, + "gpu_target_peak": 82620544, + "gpu_uniform_peak": 306225152, + "gpu_staging_peak": 69206016, + "gpu_readback_peak": 0, + "retirement_capacity": 2047, + "retirement_high_water": 296, + "admission_ns": 349722693916000, + "first_submission_ns": 349730978537708, + "admission_to_submission_ns": 8284621708 + }, + { + "generation": 4, + "cpu_live": 1114272, + "cpu_peak": 2818208, + "staging_live": 160, + "staging_peak": 32928, + "metadata_live": 144, + "metadata_blocks": 1, + "gpu_live": 1490615208, + "gpu_peak": 1532969812, + "retirement_queued": 0, + "retirement_draining": 0, + "cache_closures": 0, + "cache_leases": 0, + "cache_retiring": 0, + "cache_asset_versions": 0, + "gpu_admitted_peak": 1532973184, + "gpu_geometry_peak": 1077018624, + "gpu_texture_peak": 0, + "gpu_target_peak": 82620544, + "gpu_uniform_peak": 306225152, + "gpu_staging_peak": 69206016, + "gpu_readback_peak": 0, + "retirement_capacity": 2047, + "retirement_high_water": 296, + "admission_ns": 349732526698541, + "first_submission_ns": 349740823762625, + "admission_to_submission_ns": 8297064084 + } + ], + "lifecycle_cycles": 4, + "cancellation_timings": [ + { + "cancelled": 1, + "boundary": "cpu", + "request_ns": 349722687420208, + "cleanup_ns": 349722687476916, + "elapsed_ns": 56708 + }, + { + "cancelled": 2, + "boundary": "gpu", + "request_ns": 349732500383833, + "cleanup_ns": 349732513550875, + "elapsed_ns": 13167042 + } + ], + "validation_enabled": true, + "source_audit_zero": true +} diff --git a/tests/scene/evidence/capacity/capacity-cancellation-8192-20261117.tar.gz b/tests/scene/evidence/capacity/capacity-cancellation-8192-20261117.tar.gz new file mode 100644 index 000000000..141932338 Binary files /dev/null and b/tests/scene/evidence/capacity/capacity-cancellation-8192-20261117.tar.gz differ diff --git a/tests/scene/evidence/capacity/capacity-current-20261117.tar.gz b/tests/scene/evidence/capacity/capacity-current-20261117.tar.gz new file mode 100644 index 000000000..77b097981 Binary files /dev/null and b/tests/scene/evidence/capacity/capacity-current-20261117.tar.gz differ diff --git a/tests/scene/evidence/capacity/capacity-metal-delegate-20261117.tar.gz b/tests/scene/evidence/capacity/capacity-metal-delegate-20261117.tar.gz new file mode 100644 index 000000000..815de3d55 Binary files /dev/null and b/tests/scene/evidence/capacity/capacity-metal-delegate-20261117.tar.gz differ diff --git a/tests/scene/evidence/capacity/draw-cap-refusal-20261117.tar.gz b/tests/scene/evidence/capacity/draw-cap-refusal-20261117.tar.gz new file mode 100644 index 000000000..9a2574d8b Binary files /dev/null and b/tests/scene/evidence/capacity/draw-cap-refusal-20261117.tar.gz differ diff --git a/tests/scene/evidence/capacity/draw-refusal-current-20261117.tar.gz b/tests/scene/evidence/capacity/draw-refusal-current-20261117.tar.gz new file mode 100644 index 000000000..55d28b59d Binary files /dev/null and b/tests/scene/evidence/capacity/draw-refusal-current-20261117.tar.gz differ diff --git a/tests/scene/evidence/capacity/draw-refusal-metal-delegate-20261117.tar.gz b/tests/scene/evidence/capacity/draw-refusal-metal-delegate-20261117.tar.gz new file mode 100644 index 000000000..2a21c6fd1 Binary files /dev/null and b/tests/scene/evidence/capacity/draw-refusal-metal-delegate-20261117.tar.gz differ diff --git a/tests/scene/evidence/capacity/material-variety-8192-20261117.tar.gz b/tests/scene/evidence/capacity/material-variety-8192-20261117.tar.gz new file mode 100644 index 000000000..97c1b1cca Binary files /dev/null and b/tests/scene/evidence/capacity/material-variety-8192-20261117.tar.gz differ diff --git a/tests/scene/evidence/capacity/texture-lease-regression-20261117.tar.gz b/tests/scene/evidence/capacity/texture-lease-regression-20261117.tar.gz new file mode 100644 index 000000000..a04c1c0e9 Binary files /dev/null and b/tests/scene/evidence/capacity/texture-lease-regression-20261117.tar.gz differ diff --git a/tests/scene/evidence/capacity/texture-overlap-capacity-20261117.tar.gz b/tests/scene/evidence/capacity/texture-overlap-capacity-20261117.tar.gz new file mode 100644 index 000000000..b4ce69c60 Binary files /dev/null and b/tests/scene/evidence/capacity/texture-overlap-capacity-20261117.tar.gz differ diff --git a/tests/scene/evidence/capacity/texture-refusal-metal-delegate-20261117.tar.gz b/tests/scene/evidence/capacity/texture-refusal-metal-delegate-20261117.tar.gz new file mode 100644 index 000000000..edde96356 Binary files /dev/null and b/tests/scene/evidence/capacity/texture-refusal-metal-delegate-20261117.tar.gz differ diff --git a/tests/scene/evidence/capacity/texture-refusal-retry-20261117.tar.gz b/tests/scene/evidence/capacity/texture-refusal-retry-20261117.tar.gz new file mode 100644 index 000000000..40f879310 Binary files /dev/null and b/tests/scene/evidence/capacity/texture-refusal-retry-20261117.tar.gz differ diff --git a/tests/scene/evidence/capacity/texture-typed-refusal-20261117.tar.gz b/tests/scene/evidence/capacity/texture-typed-refusal-20261117.tar.gz new file mode 100644 index 000000000..7ae628e4f Binary files /dev/null and b/tests/scene/evidence/capacity/texture-typed-refusal-20261117.tar.gz differ diff --git a/tests/scene/evidence/failures/package-failures-current-20261117.txt b/tests/scene/evidence/failures/package-failures-current-20261117.txt new file mode 100644 index 000000000..8d780db8f --- /dev/null +++ b/tests/scene/evidence/failures/package-failures-current-20261117.txt @@ -0,0 +1,18 @@ +[==========] Running 1 test from 1 test suite. +[----------] Global test environment set-up. +[----------] 1 test from SceneCookProcessTest +[ RUN ] SceneCookProcessTest.ExecutesRequestedPhase +Cook peak working bytes: 12051080 +ZE_COOK_FIXTURE_WRITTEN +[ OK ] SceneCookProcessTest.ExecutesRequestedPhase (60 ms) +[----------] 1 test from SceneCookProcessTest (60 ms total) + +[----------] Global test environment tear-down +[==========] 1 test from 1 test suite ran. (60 ms total) +[ PASSED ] 1 test. +Cooked asset fixture submitted eight frames with one draw and shut down normally. +Rejected incorrect expected draw count. +Rejected missing cooked package. +Rejected corrupt cooked package. +Rejected target cooked package. +Rejected format cooked package. diff --git a/tests/scene/evidence/failures/standalone-package-failures-20261117.tar.gz b/tests/scene/evidence/failures/standalone-package-failures-20261117.tar.gz new file mode 100644 index 000000000..5a0b56130 Binary files /dev/null and b/tests/scene/evidence/failures/standalone-package-failures-20261117.tar.gz differ diff --git a/tests/scene/evidence/lifecycle/cancellation-handoff-100-20261117.tar.gz b/tests/scene/evidence/lifecycle/cancellation-handoff-100-20261117.tar.gz new file mode 100644 index 000000000..b04080a6c Binary files /dev/null and b/tests/scene/evidence/lifecycle/cancellation-handoff-100-20261117.tar.gz differ diff --git a/tests/scene/evidence/lifecycle/cancellation-mailbox-diagnostic-20261117.tar.gz b/tests/scene/evidence/lifecycle/cancellation-mailbox-diagnostic-20261117.tar.gz new file mode 100644 index 000000000..99699ab57 Binary files /dev/null and b/tests/scene/evidence/lifecycle/cancellation-mailbox-diagnostic-20261117.tar.gz differ diff --git a/tests/scene/evidence/lifecycle/cancellation-no-frame-20261117.tar.gz b/tests/scene/evidence/lifecycle/cancellation-no-frame-20261117.tar.gz new file mode 100644 index 000000000..218259c28 Binary files /dev/null and b/tests/scene/evidence/lifecycle/cancellation-no-frame-20261117.tar.gz differ diff --git a/tests/scene/evidence/lifecycle/cancellation-preacquire-100-20261117.tar.gz b/tests/scene/evidence/lifecycle/cancellation-preacquire-100-20261117.tar.gz new file mode 100644 index 000000000..e102b736b Binary files /dev/null and b/tests/scene/evidence/lifecycle/cancellation-preacquire-100-20261117.tar.gz differ diff --git a/tests/scene/evidence/lifecycle/cancellation-preacquire-20261117.tar.gz b/tests/scene/evidence/lifecycle/cancellation-preacquire-20261117.tar.gz new file mode 100644 index 000000000..7b3345868 Binary files /dev/null and b/tests/scene/evidence/lifecycle/cancellation-preacquire-20261117.tar.gz differ diff --git a/tests/scene/evidence/lifecycle/lifecycle-cancellation-1100-20261117.json.gz b/tests/scene/evidence/lifecycle/lifecycle-cancellation-1100-20261117.json.gz new file mode 100644 index 000000000..527e3e688 Binary files /dev/null and b/tests/scene/evidence/lifecycle/lifecycle-cancellation-1100-20261117.json.gz differ diff --git a/tests/scene/evidence/lifecycle/lifecycle-cancellation-1100-logs-20261117.tar.gz b/tests/scene/evidence/lifecycle/lifecycle-cancellation-1100-logs-20261117.tar.gz new file mode 100644 index 000000000..d2d8e31b0 Binary files /dev/null and b/tests/scene/evidence/lifecycle/lifecycle-cancellation-1100-logs-20261117.tar.gz differ diff --git a/tests/scene/evidence/lifecycle/lifecycle-final-1100-20261117.tar.gz b/tests/scene/evidence/lifecycle/lifecycle-final-1100-20261117.tar.gz new file mode 100644 index 000000000..e4c2ec8bb Binary files /dev/null and b/tests/scene/evidence/lifecycle/lifecycle-final-1100-20261117.tar.gz differ diff --git a/tests/scene/evidence/performance/macos-drawable-stack-20261117.txt.gz b/tests/scene/evidence/performance/macos-drawable-stack-20261117.txt.gz new file mode 100644 index 000000000..698f15ac0 Binary files /dev/null and b/tests/scene/evidence/performance/macos-drawable-stack-20261117.txt.gz differ diff --git a/tests/scene/evidence/performance/sampleproject-30-20261117.tar.gz b/tests/scene/evidence/performance/sampleproject-30-20261117.tar.gz new file mode 100644 index 000000000..b5ba08522 Binary files /dev/null and b/tests/scene/evidence/performance/sampleproject-30-20261117.tar.gz differ diff --git a/tests/scene/evidence/performance/sampleproject-corrected-domains-20261117.tar.gz b/tests/scene/evidence/performance/sampleproject-corrected-domains-20261117.tar.gz new file mode 100644 index 000000000..e6223dc1c Binary files /dev/null and b/tests/scene/evidence/performance/sampleproject-corrected-domains-20261117.tar.gz differ diff --git a/tests/scene/evidence/performance/sampleproject-occlusion-transition-20261117.tar.gz b/tests/scene/evidence/performance/sampleproject-occlusion-transition-20261117.tar.gz new file mode 100644 index 000000000..90af8dcdb Binary files /dev/null and b/tests/scene/evidence/performance/sampleproject-occlusion-transition-20261117.tar.gz differ diff --git a/tests/scene/evidence/performance/sampleproject-post-steady-20261117.tar.gz b/tests/scene/evidence/performance/sampleproject-post-steady-20261117.tar.gz new file mode 100644 index 000000000..22ca804ec Binary files /dev/null and b/tests/scene/evidence/performance/sampleproject-post-steady-20261117.tar.gz differ diff --git a/tests/scene/evidence/performance/sampleproject-triple-drawable-20261117.tar.gz b/tests/scene/evidence/performance/sampleproject-triple-drawable-20261117.tar.gz new file mode 100644 index 000000000..ad746d3f6 Binary files /dev/null and b/tests/scene/evidence/performance/sampleproject-triple-drawable-20261117.tar.gz differ diff --git a/tests/scene/evidence/performance/sampleproject-visible-final-20261117.tar.gz b/tests/scene/evidence/performance/sampleproject-visible-final-20261117.tar.gz new file mode 100644 index 000000000..b4aee93c3 Binary files /dev/null and b/tests/scene/evidence/performance/sampleproject-visible-final-20261117.tar.gz differ diff --git a/tests/scene/evidence/performance/sampleproject-visible-user-20261117.tar.gz b/tests/scene/evidence/performance/sampleproject-visible-user-20261117.tar.gz new file mode 100644 index 000000000..ba71ba0bf Binary files /dev/null and b/tests/scene/evidence/performance/sampleproject-visible-user-20261117.tar.gz differ diff --git a/tests/scene/evidence/performance/sampleproject-window-visibility-20261117.tar.gz b/tests/scene/evidence/performance/sampleproject-window-visibility-20261117.tar.gz new file mode 100644 index 000000000..9c1f1f10b Binary files /dev/null and b/tests/scene/evidence/performance/sampleproject-window-visibility-20261117.tar.gz differ diff --git a/tests/scene/evidence/performance/submit-async-probe-20261117.json.gz b/tests/scene/evidence/performance/submit-async-probe-20261117.json.gz new file mode 100644 index 000000000..2929a2598 Binary files /dev/null and b/tests/scene/evidence/performance/submit-async-probe-20261117.json.gz differ diff --git a/tests/scene/evidence/performance/submit-diagnostic-20261117.json b/tests/scene/evidence/performance/submit-diagnostic-20261117.json new file mode 100644 index 000000000..d9f34c333 --- /dev/null +++ b/tests/scene/evidence/performance/submit-diagnostic-20261117.json @@ -0,0 +1,726 @@ +{ + "scope": "diagnostic comparison; not matched-frame attribution or qualification", + "manifest_sha256": "3d03885391f97d37151e890ee7de525251474e58420413c8cf22d0376afe9b22", + "filesystem_cache_conditions": "uncontrolled operating-system cache; no cold-disk claim", + "initial_trace_starts_after_admission_ns": 0, + "initial_admission_marker_observed": true, + "steady_submitted_frames": 120, + "windows": { + "initial_load": { + "start_ns": 349421608604583, + "end_ns": 349428143618791, + "first_record_ns": 349421608708750, + "main_frame_diagnostics": { + "frames": 1922, + "partial_scopes": 0, + "unassigned_scopes": [], + "runtime_update_ms": { + "count": 1922, + "p50_ms": 0.001708, + "p95_ms": 0.004209, + "p99_ms": 0.007208, + "max_ms": 0.291708, + "worst_frame_end_ns": 349427510985000 + }, + "guarded_commit_ms": { + "count": 1, + "p50_ms": 0.0015, + "p95_ms": 0.0015, + "p99_ms": 0.0015, + "max_ms": 0.0015, + "worst_frame_end_ns": 349428128957291 + } + }, + "render_frame_diagnostics": { + "frames": 647, + "partial_scopes": 2, + "unassigned_scopes": [ + { + "stage": "SceneLoad.Render.EnvironmentPreparation", + "end_ns": 349428136293625, + "duration_ms": 0.004333 + }, + { + "stage": "SceneLoad.Render.EnvironmentPreparation", + "end_ns": 349428136740416, + "duration_ms": 0.002458 + } + ], + "cooked_upload_ms": { + "count": 335, + "p50_ms": 0.023959, + "p95_ms": 0.138875, + "p99_ms": 0.188459, + "max_ms": 0.234583, + "worst_frame_end_ns": 349423608223541 + }, + "environment_preparation_ms": { + "count": 647, + "p50_ms": 0.0037089999999999996, + "p95_ms": 0.240917, + "p99_ms": 1.817459, + "max_ms": 3.988625, + "worst_frame_end_ns": 349426543536291 + } + }, + "stages": { + "SceneLoad.Main.FrameWorkMs": { + "count": 1922, + "p50_ms": 0.081584, + "p95_ms": 0.1895, + "p99_ms": 0.302667, + "max_ms": 50.13025 + }, + "SceneLoad.Main.FrameIntervalMs": { + "count": 1922, + "p50_ms": 3.333916, + "p95_ms": 3.357208, + "p99_ms": 4.451041, + "max_ms": 50.13075 + }, + "SceneLoad.Main.RuntimeUpdate": { + "count": 1922, + "p50_ms": 0.001708, + "p95_ms": 0.004209, + "p99_ms": 0.007208, + "max_ms": 0.291708 + }, + "SceneLoad.Main.PlatformEvents": { + "count": 1922, + "p50_ms": 0.018, + "p95_ms": 0.041209, + "p99_ms": 0.155625, + "max_ms": 49.986917 + }, + "SceneLoad.Main.OverlayBuild": { + "count": 1922, + "p50_ms": 0.0005, + "p95_ms": 0.0555, + "p99_ms": 0.08525, + "max_ms": 0.199625 + }, + "SceneLoad.Main.SceneSynchronization": { + "count": 1616, + "p50_ms": 0.027333, + "p95_ms": 0.060084, + "p99_ms": 0.066333, + "max_ms": 0.180291 + }, + "SceneLoad.Main.PrepareScene": { + "count": 1922, + "p50_ms": 0.00025, + "p95_ms": 0.000542, + "p99_ms": 0.00075, + "max_ms": 0.014125 + }, + "SceneLoad.Main.PublishFrame": { + "count": 1922, + "p50_ms": 0.003166, + "p95_ms": 0.007, + "p99_ms": 0.010208, + "max_ms": 0.040583 + }, + "SceneLoad.Main.RuntimeGuardedCommit": { + "count": 1, + "p50_ms": 0.0015, + "p95_ms": 0.0015, + "p99_ms": 0.0015, + "max_ms": 0.0015 + }, + "SceneLoad.Render.FrameWorkMs": { + "count": 647, + "p50_ms": 8.324166, + "p95_ms": 9.057958, + "p99_ms": 16.205917, + "max_ms": 1008.485541 + }, + "SceneLoad.Render.FrameIntervalMs": { + "count": 647, + "p50_ms": 8.327708, + "p95_ms": 9.085875, + "p99_ms": 16.650625, + "max_ms": 1008.490541 + }, + "SceneLoad.Render.CookedUploadVisit": { + "count": 335, + "p50_ms": 0.023959, + "p95_ms": 0.138875, + "p99_ms": 0.188459, + "max_ms": 0.234583 + }, + "SceneLoad.Render.EnvironmentPreparation": { + "count": 1172, + "p50_ms": 0.002375, + "p95_ms": 0.115959, + "p99_ms": 1.030458, + "max_ms": 2.711083 + }, + "SceneLoad.Render.QueueSubmit": { + "count": 648, + "p50_ms": 4.738667, + "p95_ms": 7.244041, + "p99_ms": 7.961584, + "max_ms": 1006.154417 + }, + "SceneLoad.Render.AcquireImage": { + "count": 648, + "p50_ms": 0.090458, + "p95_ms": 0.167666, + "p99_ms": 0.41075, + "max_ms": 2.131541 + }, + "SceneLoad.Render.PresentWSI": { + "count": 647, + "p50_ms": 0.067583, + "p95_ms": 8.137792, + "p99_ms": 8.518333, + "max_ms": 15.869125 + }, + "SceneLoad.Render.RecordScene": { + "count": 524, + "p50_ms": 1.764125, + "p95_ms": 3.206584, + "p99_ms": 4.365875, + "max_ms": 9.986292 + } + }, + "render_interval_outliers": { + "threshold_ms": 33.613125, + "count": 2, + "largest_intervals": [ + { + "interval_end_ns": 349426075807833, + "interval_ms": 1008.490541, + "longest_overlapping_scopes": [ + { + "name": "SceneLoad.Render.EndFrame", + "end_ns": 349426075807083, + "duration_ms": 1006.535625 + }, + { + "name": "SceneLoad.Render.SubmitAndPresent", + "end_ns": 349426075805791, + "duration_ms": 1006.5305 + }, + { + "name": "SceneLoad.Render.QueueSubmit", + "end_ns": 349426075581333, + "duration_ms": 1006.154417 + }, + { + "name": "SceneLoad.Render.RecordScene", + "end_ns": 349425069271041, + "duration_ms": 1.491542 + }, + { + "name": "SceneLoad.Render.GraphExecute", + "end_ns": 349425069267916, + "duration_ms": 1.460667 + } + ], + "overlapping_platform_events": [ + { + "end_ns": 349425078731875, + "duration_ms": 5.854125, + "overlap_ms": 5.854125 + }, + { + "end_ns": 349425183396916, + "duration_ms": 4.54225, + "overlap_ms": 4.54225 + }, + { + "end_ns": 349425666618583, + "duration_ms": 0.667458, + "overlap_ms": 0.667458 + }, + { + "end_ns": 349425145552333, + "duration_ms": 0.058458, + "overlap_ms": 0.058458 + }, + { + "end_ns": 349425085537916, + "duration_ms": 0.056917, + "overlap_ms": 0.056917 + } + ], + "recording_worker_samples": [ + { + "name": "SceneLoad.RecordingWorker.DispatchDelayMs", + "timestamp_ns": 349425068897333, + "value_ms": 0.013875 + }, + { + "name": "SceneLoad.RecordingWorker.DispatchDelayMs", + "timestamp_ns": 349425068935041, + "value_ms": 0.012542 + }, + { + "name": "SceneLoad.RecordingWorker.DispatchDelayMs", + "timestamp_ns": 349425068745458, + "value_ms": 0.011 + }, + { + "name": "SceneLoad.RecordingWorker.DispatchDelayMs", + "timestamp_ns": 349425068435291, + "value_ms": 0.010125 + }, + { + "name": "SceneLoad.RecordingWorker.DispatchDelayMs", + "timestamp_ns": 349425068635625, + "value_ms": 0.008166 + }, + { + "name": "SceneLoad.RecordingWorker.DispatchDelayMs", + "timestamp_ns": 349425068987791, + "value_ms": 0.00775 + }, + { + "name": "SceneLoad.RecordingWorker.DispatchDelayMs", + "timestamp_ns": 349425068672250, + "value_ms": 0.006791 + }, + { + "name": "SceneLoad.RecordingWorker.DispatchDelayMs", + "timestamp_ns": 349425068461083, + "value_ms": 0.002584 + } + ], + "queue_submit_samples": [ + { + "name": "SceneLoad.QueueSubmit.LongWallMs", + "timestamp_ns": 349426075580875, + "value_ms": 1006.152791 + }, + { + "name": "SceneLoad.QueueSubmit.LongCpuMs", + "timestamp_ns": 349426075581000, + "value_ms": 1.202458 + }, + { + "name": "SceneLoad.QueueSubmit.LongNonCpuMs", + "timestamp_ns": 349426075581125, + "value_ms": 1004.950333 + } + ] + }, + { + "interval_end_ns": 349421660603208, + "interval_ms": 51.929083, + "longest_overlapping_scopes": [ + { + "name": "SceneLoad.Render.EndFrame", + "end_ns": 349421660602666, + "duration_ms": 1.278625 + }, + { + "name": "SceneLoad.Render.SubmitAndPresent", + "end_ns": 349421660601916, + "duration_ms": 1.246125 + }, + { + "name": "SceneLoad.Render.PresentWSI", + "end_ns": 349421660600416, + "duration_ms": 1.194 + }, + { + "name": "SceneLoad.Render.BeginFrame", + "end_ns": 349421659323416, + "duration_ms": 0.274833 + }, + { + "name": "SceneLoad.Render.ResizePreparation", + "end_ns": 349421659048291, + "duration_ms": 0.113625 + } + ], + "overlapping_platform_events": [ + { + "end_ns": 349421658697333, + "duration_ms": 49.986917, + "overlap_ms": 49.986917 + }, + { + "end_ns": 349421658845541, + "duration_ms": 0.006042, + "overlap_ms": 0.006042 + } + ], + "recording_worker_samples": [], + "queue_submit_samples": [] + } + ], + "note": "Overlap is diagnostic only; nested scopes are not additive and do not establish causation." + } + }, + "steady": { + "start_ns": 349428255352333, + "end_ns": 349429664199083, + "first_record_ns": 349428255352541, + "main_frame_diagnostics": { + "frames": 397, + "partial_scopes": 0, + "unassigned_scopes": [], + "runtime_update_ms": { + "count": 397, + "p50_ms": 0.002167, + "p95_ms": 0.0035, + "p99_ms": 0.005667, + "max_ms": 0.06, + "worst_frame_end_ns": 349428255460041 + }, + "guarded_commit_ms": null + }, + "render_frame_diagnostics": { + "frames": 120, + "partial_scopes": 2, + "unassigned_scopes": [ + { + "stage": "SceneLoad.Render.EnvironmentPreparation", + "end_ns": 349429661381458, + "duration_ms": 0.001833 + }, + { + "stage": "SceneLoad.Render.EnvironmentPreparation", + "end_ns": 349429661549125, + "duration_ms": 0.001292 + } + ], + "cooked_upload_ms": null, + "environment_preparation_ms": { + "count": 119, + "p50_ms": 0.0072499999999999995, + "p95_ms": 0.011333, + "p99_ms": 0.022874, + "max_ms": 0.030334, + "worst_frame_end_ns": 349428795574750 + } + }, + "stages": { + "SceneLoad.Main.FrameWorkMs": { + "count": 397, + "p50_ms": 0.124917, + "p95_ms": 0.233459, + "p99_ms": 0.303666, + "max_ms": 0.458125 + }, + "SceneLoad.Main.FrameIntervalMs": { + "count": 397, + "p50_ms": 3.334208, + "p95_ms": 4.062666, + "p99_ms": 9.310417, + "max_ms": 13.250833 + }, + "SceneLoad.Main.RuntimeUpdate": { + "count": 397, + "p50_ms": 0.002167, + "p95_ms": 0.0035, + "p99_ms": 0.005667, + "max_ms": 0.06 + }, + "SceneLoad.Main.PlatformEvents": { + "count": 397, + "p50_ms": 0.02475, + "p95_ms": 0.04975, + "p99_ms": 0.065875, + "max_ms": 0.075167 + }, + "SceneLoad.Main.OverlayBuild": { + "count": 397, + "p50_ms": 0.000625, + "p95_ms": 0.064542, + "p99_ms": 0.091833, + "max_ms": 0.156042 + }, + "SceneLoad.Main.SceneSynchronization": { + "count": 397, + "p50_ms": 0.039417, + "p95_ms": 0.060292, + "p99_ms": 0.102416, + "max_ms": 0.284667 + }, + "SceneLoad.Main.PrepareScene": { + "count": 397, + "p50_ms": 0.011042, + "p95_ms": 0.018541, + "p99_ms": 0.028459, + "max_ms": 0.050709 + }, + "SceneLoad.Main.PublishFrame": { + "count": 397, + "p50_ms": 0.005542, + "p95_ms": 0.011, + "p99_ms": 0.018125, + "max_ms": 0.032875 + }, + "SceneLoad.Render.FrameWorkMs": { + "count": 120, + "p50_ms": 8.455791, + "p95_ms": 25.6105, + "p99_ms": 25.908625, + "max_ms": 26.299 + }, + "SceneLoad.Render.FrameIntervalMs": { + "count": 120, + "p50_ms": 8.46275, + "p95_ms": 25.613125, + "p99_ms": 25.911709, + "max_ms": 26.303584 + }, + "SceneLoad.Render.EnvironmentPreparation": { + "count": 240, + "p50_ms": 0.003167, + "p95_ms": 0.00575, + "p99_ms": 0.012416, + "max_ms": 0.02825 + }, + "SceneLoad.Render.QueueSubmit": { + "count": 120, + "p50_ms": 3.861417, + "p95_ms": 23.640791, + "p99_ms": 23.874833, + "max_ms": 24.001625 + }, + "SceneLoad.Render.AcquireImage": { + "count": 120, + "p50_ms": 0.140292, + "p95_ms": 0.667417, + "p99_ms": 4.849708, + "max_ms": 7.24625 + }, + "SceneLoad.Render.PresentWSI": { + "count": 120, + "p50_ms": 0.076541, + "p95_ms": 0.10075, + "p99_ms": 0.146291, + "max_ms": 0.2035 + }, + "SceneLoad.Render.RecordScene": { + "count": 120, + "p50_ms": 3.138167, + "p95_ms": 4.177125, + "p99_ms": 5.886833, + "max_ms": 6.222 + } + } + }, + "replacement": { + "start_ns": 349429664218583, + "end_ns": 349436627879583, + "first_record_ns": 349429664219458, + "main_frame_diagnostics": { + "frames": 2052, + "partial_scopes": 0, + "unassigned_scopes": [], + "runtime_update_ms": { + "count": 2052, + "p50_ms": 0.002375, + "p95_ms": 0.004333, + "p99_ms": 0.010209, + "max_ms": 0.20275, + "worst_frame_end_ns": 349436595155666 + }, + "guarded_commit_ms": { + "count": 1, + "p50_ms": 0.001291, + "p95_ms": 0.001291, + "p99_ms": 0.001291, + "max_ms": 0.001291, + "worst_frame_end_ns": 349436595155666 + } + }, + "render_frame_diagnostics": { + "frames": 579, + "partial_scopes": 2, + "unassigned_scopes": [ + { + "stage": "SceneLoad.Render.EnvironmentPreparation", + "end_ns": 349436603115250, + "duration_ms": 0.0025 + }, + { + "stage": "SceneLoad.Render.EnvironmentPreparation", + "end_ns": 349436603285916, + "duration_ms": 0.001 + } + ], + "cooked_upload_ms": { + "count": 335, + "p50_ms": 0.051458, + "p95_ms": 0.173834, + "p99_ms": 0.234209, + "max_ms": 0.272708, + "worst_frame_end_ns": 349432399915791 + }, + "environment_preparation_ms": { + "count": 578, + "p50_ms": 0.008166, + "p95_ms": 0.152333, + "p99_ms": 0.294458, + "max_ms": 0.432625, + "worst_frame_end_ns": 349434667404958 + } + }, + "stages": { + "SceneLoad.Main.FrameWorkMs": { + "count": 2052, + "p50_ms": 0.115167, + "p95_ms": 0.231459, + "p99_ms": 0.32025, + "max_ms": 0.536542 + }, + "SceneLoad.Main.FrameIntervalMs": { + "count": 2051, + "p50_ms": 3.334041, + "p95_ms": 3.37175, + "p99_ms": 4.093416, + "max_ms": 13.251083 + }, + "SceneLoad.Main.RuntimeUpdate": { + "count": 2052, + "p50_ms": 0.002375, + "p95_ms": 0.004333, + "p99_ms": 0.010209, + "max_ms": 0.20275 + }, + "SceneLoad.Main.PlatformEvents": { + "count": 2051, + "p50_ms": 0.02175, + "p95_ms": 0.046333, + "p99_ms": 0.072292, + "max_ms": 0.174625 + }, + "SceneLoad.Main.OverlayBuild": { + "count": 2052, + "p50_ms": 0.000583, + "p95_ms": 0.063875, + "p99_ms": 0.0985, + "max_ms": 0.398667 + }, + "SceneLoad.Main.SceneSynchronization": { + "count": 2052, + "p50_ms": 0.03775, + "p95_ms": 0.05725, + "p99_ms": 0.093792, + "max_ms": 0.191167 + }, + "SceneLoad.Main.PrepareScene": { + "count": 2052, + "p50_ms": 0.010208, + "p95_ms": 0.015667, + "p99_ms": 0.024583, + "max_ms": 0.211125 + }, + "SceneLoad.Main.PublishFrame": { + "count": 2052, + "p50_ms": 0.005125, + "p95_ms": 0.008625, + "p99_ms": 0.014375, + "max_ms": 0.029209 + }, + "SceneLoad.Main.RuntimeGuardedCommit": { + "count": 1, + "p50_ms": 0.001291, + "p95_ms": 0.001291, + "p99_ms": 0.001291, + "max_ms": 0.001291 + }, + "SceneLoad.Render.FrameWorkMs": { + "count": 579, + "p50_ms": 8.500042, + "p95_ms": 25.614416, + "p99_ms": 25.968209, + "max_ms": 26.3605 + }, + "SceneLoad.Render.FrameIntervalMs": { + "count": 579, + "p50_ms": 8.506959, + "p95_ms": 25.616416, + "p99_ms": 25.970875, + "max_ms": 26.364917 + }, + "SceneLoad.Render.CookedUploadVisit": { + "count": 335, + "p50_ms": 0.051458, + "p95_ms": 0.173834, + "p99_ms": 0.234209, + "max_ms": 0.272708 + }, + "SceneLoad.Render.EnvironmentPreparation": { + "count": 1158, + "p50_ms": 0.003917, + "p95_ms": 0.066542, + "p99_ms": 0.179042, + "max_ms": 0.427791 + }, + "SceneLoad.Render.QueueSubmit": { + "count": 580, + "p50_ms": 3.845417, + "p95_ms": 23.669333, + "p99_ms": 23.823417, + "max_ms": 24.467709 + }, + "SceneLoad.Render.AcquireImage": { + "count": 579, + "p50_ms": 0.137375, + "p95_ms": 0.316125, + "p99_ms": 0.863667, + "max_ms": 6.572875 + }, + "SceneLoad.Render.PresentWSI": { + "count": 579, + "p50_ms": 0.067666, + "p95_ms": 0.095583, + "p99_ms": 0.129542, + "max_ms": 0.19975 + }, + "SceneLoad.Render.RecordScene": { + "count": 579, + "p50_ms": 3.1635, + "p95_ms": 3.98775, + "p99_ms": 7.044916, + "max_ms": 19.097791 + } + }, + "render_interval_outliers": { + "threshold_ms": 33.613125, + "count": 0, + "largest_intervals": [], + "note": "Overlap is diagnostic only; nested scopes are not additive and do not establish causation." + } + } + }, + "binary_sha256": { + "cooker_sha256": "cc4bc8c155a47c8a021030132134589bd4f7d68fc74f1568e69ff22c4294ba1f", + "runtime_sha256": "3e0bf49dbdaf11196587a2d9ba6d1f3b5af15a8a5d25ba82010aebe860ad658a" + }, + "shader_cache": { + "sha256": "dfd8d94eb2a7e0afc9e34f4644052fb2d86de3e7396cb0ff5496d89717d21916", + "files": 44 + }, + "hardware": { + "cpu": "Apple M4 Pro", + "physical_memory_bytes": 51539607552 + }, + "device": { + "gpu": "Apple M4 Pro", + "driver_path": "/usr/local/lib/libMoltenVK.dylib", + "driver_sha256": "0b398933adeb1aba84aff76ef438f54f17e7deb6960beb5711fa6e23cd192cdb" + }, + "viewport": { + "scene_viewport": [ + 1280, + 720 + ], + "window": [ + 1280, + 720 + ], + "framebuffer": [ + 2560, + 1440 + ], + "resize_after_baseline": false + } +} diff --git a/tests/scene/evidence/visual/sampleproject-approved-current-20261117.tar.gz b/tests/scene/evidence/visual/sampleproject-approved-current-20261117.tar.gz new file mode 100644 index 000000000..3339d81b5 Binary files /dev/null and b/tests/scene/evidence/visual/sampleproject-approved-current-20261117.tar.gz differ diff --git a/tests/scene/evidence/visual/sampleproject-approved-preacquire-20261117.tar.gz b/tests/scene/evidence/visual/sampleproject-approved-preacquire-20261117.tar.gz new file mode 100644 index 000000000..ec54ffb54 Binary files /dev/null and b/tests/scene/evidence/visual/sampleproject-approved-preacquire-20261117.tar.gz differ diff --git a/tests/scene/evidence/visual/sampleproject-domain-visual-20261117.tar.gz b/tests/scene/evidence/visual/sampleproject-domain-visual-20261117.tar.gz new file mode 100644 index 000000000..542ebe15a Binary files /dev/null and b/tests/scene/evidence/visual/sampleproject-domain-visual-20261117.tar.gz differ diff --git a/tests/scene/evidence/visual/sampleproject-hierarchy-20261117.tar.gz b/tests/scene/evidence/visual/sampleproject-hierarchy-20261117.tar.gz new file mode 100644 index 000000000..cb2bd8abb Binary files /dev/null and b/tests/scene/evidence/visual/sampleproject-hierarchy-20261117.tar.gz differ diff --git a/tests/scene/evidence/visual/sampleproject-hierarchy-replacement-20261117.tar.gz b/tests/scene/evidence/visual/sampleproject-hierarchy-replacement-20261117.tar.gz new file mode 100644 index 000000000..cb70f5a21 Binary files /dev/null and b/tests/scene/evidence/visual/sampleproject-hierarchy-replacement-20261117.tar.gz differ diff --git a/tests/scene/evidence/visual/sampleproject-light-control-20261117.tar.gz b/tests/scene/evidence/visual/sampleproject-light-control-20261117.tar.gz new file mode 100644 index 000000000..00f4626c9 Binary files /dev/null and b/tests/scene/evidence/visual/sampleproject-light-control-20261117.tar.gz differ diff --git a/tests/scene/evidence/visual/sampleproject-metal-delegate-20261117.tar.gz b/tests/scene/evidence/visual/sampleproject-metal-delegate-20261117.tar.gz new file mode 100644 index 000000000..e9dce726b Binary files /dev/null and b/tests/scene/evidence/visual/sampleproject-metal-delegate-20261117.tar.gz differ diff --git a/tests/scene/evidence/visual/sampleproject-provenance-20261117.tar.gz b/tests/scene/evidence/visual/sampleproject-provenance-20261117.tar.gz new file mode 100644 index 000000000..75e4b00d8 Binary files /dev/null and b/tests/scene/evidence/visual/sampleproject-provenance-20261117.tar.gz differ diff --git a/tests/scene/evidence/visual/sampleproject-provenance-repeat-20261117.tar.gz b/tests/scene/evidence/visual/sampleproject-provenance-repeat-20261117.tar.gz new file mode 100644 index 000000000..e1245d7fa Binary files /dev/null and b/tests/scene/evidence/visual/sampleproject-provenance-repeat-20261117.tar.gz differ diff --git a/tests/scene/evidence/visual/sampleproject-transparency-control-20261117.tar.gz b/tests/scene/evidence/visual/sampleproject-transparency-control-20261117.tar.gz new file mode 100644 index 000000000..7e1a033ec Binary files /dev/null and b/tests/scene/evidence/visual/sampleproject-transparency-control-20261117.tar.gz differ diff --git a/tests/scene/sampleproject_cooked_parity.py b/tests/scene/sampleproject_cooked_parity.py new file mode 100644 index 000000000..24a08a5c1 --- /dev/null +++ b/tests/scene/sampleproject_cooked_parity.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Capture a copied SampleProject scene through source and cooked paths. + +Requires a desktop Vulkan session and built SceneCookProcess and Obelisk. This +records a comparison, not a reviewed reference or a calibrated qualification. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +from compare_scene_capture import compare_region, parse_region, read_ppm, validate_capture_log + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project", required=True, type=Path) + parser.add_argument("--cooker", required=True, type=Path) + parser.add_argument("--obelisk", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--scene", help="select a scene already listed in the project, or an existing scene file by stem") + parser.add_argument("--region", action="append", type=parse_region, default=[], help="compare an important object region: name:x,y,width,height") + parser.add_argument("--minimum-assets", type=int, default=0, help="require at least this many assets in the cooked manifest") + parser.add_argument("--expected-draws", type=int, default=0, help="require this many prepared and submitted draws in each capture") + parser.add_argument("--stage-trace", action="store_true", help="write a bounded cooked SceneLoad trace in the output directory") + parser.add_argument("--timeout", type=float, default=180) + args = parser.parse_args() + project = args.project.resolve(strict=True) + cooker = args.cooker.resolve(strict=True) + obelisk = args.obelisk.resolve(strict=True) + output = args.output.resolve(strict=True) + if args.timeout <= 0 or not project.is_file() or not output.is_dir() or any(output.iterdir()): + parser.error("--project must be a file; --output an existing empty directory; --timeout positive") + if args.minimum_assets < 0 or args.expected_draws < 0 or len({name for name, _ in args.region}) != len(args.region): + parser.error("asset and draw limits must be nonnegative and region names must be unique") + if output == project.parent or project.parent in output.parents: + parser.error("--output must be outside the project workspace") + config = json.loads(project.read_text(encoding="utf-8")) + defaults = [scene["name"] for scene in config["sceneList"] if scene.get("isDefault")] + if len(defaults) != 1 or config.get("workingSpace") != ".": + raise ValueError("project must use one default scene and a project-local workspace") + # Do not let a copied workspace silently follow authoring files outside it. + for directory, names, files in os.walk(project.parent, followlinks=False): + for name in names + files: + if (Path(directory) / name).is_symlink(): + raise ValueError("project copy contains a symlink") + source_log, cooked_log = output / "source-log", output / "cooked-log" + source_log.mkdir() + cooked_log.mkdir() + source_image, cooked_image = output / "source.ppm", output / "cooked.ppm" + + with tempfile.TemporaryDirectory(prefix="renderer-sample-parity-") as temporary: + root = Path(temporary) + source = root / "project" + shutil.copytree(project.parent, source) + copied_project = source / project.name + if args.scene: + if not args.scene or Path(args.scene).name != args.scene or not (source / "Scenes" / (args.scene + ".zescene")).is_file(): + raise ValueError("selected scene must be a scene file stem in the copied SampleProject Scenes directory") + config["sceneList"] = [{"name": args.scene, "isDefault": True}] + copied_project.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + defaults = [args.scene] + selected_config_digest = hashlib.sha256(copied_project.read_bytes()).hexdigest() + workspace = root / "cook" + workspace.mkdir() + subprocess.run([str(cooker), "smoke-offline", str(copied_project), str(workspace), "805306368"], check=True, timeout=args.timeout) + package = workspace / "output" / "desktop-vulkan-le64" + manifest_bytes = (package / "manifest.json").read_bytes() + manifest = json.loads(manifest_bytes) + if len(manifest["assets"]) < args.minimum_assets: + raise ValueError("cooked package contains fewer assets than required") + selected = [scene for scene in manifest["scenes"] if Path(scene["source"]).name == defaults[0] + ".zescene"] + if len(selected) != 1: + raise ValueError("cooked manifest does not contain the default scene exactly once") + subprocess.run([str(obelisk), "--launchEditor", "true", "--projectConfigFile", str(copied_project), + "--sourceCapture", str(source_image), "--cookedExpectedDraws", str(args.expected_draws), "--logDirectory", str(source_log)], check=True, timeout=args.timeout) + validate_capture_log(source_log / "engine_dump.log") + shutil.rmtree(source) + cooked_command = [str(obelisk), "--launchEditor", "true", "--cookedPackage", str(package), + "--cookedScene", selected[0]["uuid"], "--cookedExitAfterFrame", + "--cookedCapture", str(cooked_image), "--cookedExpectedDraws", str(args.expected_draws), "--logDirectory", str(cooked_log)] + if args.stage_trace: + cooked_command.extend(["--cookedStageTrace", str(output / "cooked-stages.csv")]) + subprocess.run(cooked_command, check=True, timeout=args.timeout) + validate_capture_log(cooked_log / "engine_dump.log") + + reference, candidate = read_ppm(source_image), read_ppm(cooked_image) + width, height = reference[:2] + comparison = compare_region(reference, candidate, (0, 0, width, height)) + regions = {name: compare_region(reference, candidate, rectangle) for name, rectangle in args.region} + stages = None + if args.stage_trace: + from validate_scene_lifecycle import read_stages + all_stages = read_stages(output / "cooked-stages.csv") + required_stages = ("SceneLoad.Worker.CookedRequest", "SceneLoad.Render.CookedUploadVisit") + if any(name not in all_stages for name in required_stages): + raise ValueError("cooked stage trace is missing worker or render upload observations") + stages = {name: result for name, result in all_stages.items() + if ".Cooked" in name or name == "SceneLoad.Render.EnvironmentPreparation"} + report = {"scope": "SampleProject scene source-versus-cooked smoke; reference review required", + "project_config_sha256": hashlib.sha256(project.read_bytes()).hexdigest(), + "selected_config_sha256": selected_config_digest, + "scene_name": defaults[0], + "manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(), + "scene_uuid": selected[0]["uuid"], "cooked_assets": len(manifest["assets"]), + "minimum_assets": args.minimum_assets, "expected_draws": args.expected_draws, + "width": width, "height": height, "viewport": comparison, "regions": regions, + "cooked_stages": stages} + (output / "comparison.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2)) + return 0 if comparison["passed"] and all(region["passed"] for region in regions.values()) else 1 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError, KeyError, subprocess.SubprocessError) as error: + print(f"SampleProject parity failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/tests/scene/sampleproject_cooked_runtime.py b/tests/scene/sampleproject_cooked_runtime.py new file mode 100644 index 000000000..ea70ec56d --- /dev/null +++ b/tests/scene/sampleproject_cooked_runtime.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""Compare source and standalone cooked captures of SampleProject level01. + +The copied scene receives a fixed authored camera; the project is never edited. +This is a render smoke test, not an approved visual reference or qualification. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile + +from compare_scene_capture import compare_region, parse_region, read_ppm, validate_capture_log_text +from cooked_capture_identity import device_identity, shader_identity +from cooked_scene_smoke import read_console_log, read_process_logs, require_clean_vulkan_console, require_zero_source_audit, sha256_file, validation_layer_status_text + +CAMERA = "b1b1b1b1-7171-4141-8181-b1b1b1b1b1b1" +SCENE = "f6d4dbea-c21d-453d-ae91-79187dfa8b84" +ALTERNATE_SCENE = "f6d4dbea-c21d-453d-ae91-79187dfa8b85" +HIERARCHY_PARENT = "beb33379-561e-4413-af36-b7d9f5a6be26" +HIERARCHY_MESH = "e97f8f91-1fe7-415a-89ec-9e67e04ad960" +CAMERA_ENTITY = f''' - "uuid": "{CAMERA}" + "components": + "CameraComponent": + "schema_version": 1 + "fields": + "fov_y": 60 + "near": 0.1 + "far": 1000 + "aspect_ratio": 1.7777778 + "is_main": true + "NameComponent": + "schema_version": 1 + "fields": + "value": "Runtime qualification camera" + "TransformComponent": + "schema_version": 1 + "fields": + "position": [0.5, 2, 6] + "rotation": [0, 0, 0] + "scale": [1, 1, 1] +''' + + +def add_sampleproject_hierarchy(scene): + """Move the mesh translation to its existing parent in a copied level01.""" + parent_start = f' - "uuid": "{HIERARCHY_PARENT}"\n'.encode() + mesh_start = f' - "uuid": "{HIERARCHY_MESH}"\n'.encode() + if scene.count(parent_start) != 1 or scene.count(mesh_start) != 1 or b'"ParentComponent"' in scene: + raise ValueError("SampleProject scene cannot safely receive the hierarchy fixture") + parent_index, mesh_index = scene.index(parent_start), scene.index(mesh_start) + if parent_index >= mesh_index: + raise ValueError("unexpected SampleProject hierarchy fixture order") + parent_end = scene.index(b' - "uuid": ', parent_index + len(parent_start)) + parent = scene[parent_index:parent_end] + mesh = scene[mesh_index:] + origin = b'"position": [0, 0, 0]' + translation = b'"position": [0.40000004, 1.2, 0]' + components = b' "components":\n' + if parent.count(origin) != 1 or mesh.count(translation) != 1 or mesh.count(components) != 1: + raise ValueError("unexpected SampleProject transform in hierarchy fixture") + parent = parent.replace(origin, translation, 1) + mesh = mesh.replace(translation, origin, 1).replace(components, components + + (f' "ParentComponent":\n "schema_version": 1\n "fields":\n "parent_uuid": "{HIERARCHY_PARENT}"\n').encode(), 1) + return scene[:parent_index] + parent + scene[parent_end:mesh_index] + mesh + + +def remove_sampleproject_directional_light(scene): + """Disable the copied scene's authored directional light for a control.""" + light = b' "LightComponent":\n' + intensity = b' "intensity": 3\n' + if scene.count(light) != 1 or scene.count(intensity) != 1: + raise ValueError("SampleProject directional light cannot be changed safely") + return scene.replace(intensity, b' "intensity": 0\n', 1) + + +def force_opaque_materials(directory): + """Change copied authored blend materials to opaque for a negative control.""" + changed = [] + for path in sorted(directory.rglob("*.zematerial")): + original = path.read_bytes() + material = json.loads(original) + if material.get("alpha_mode") == "BLEND": + material["alpha_mode"] = "OPAQUE" + modified = (json.dumps(material, indent=4) + "\n").encode() + path.write_bytes(modified) + changed.append({"path": path.relative_to(directory).as_posix(), + "source_sha256": hashlib.sha256(original).hexdigest(), + "modified_sha256": hashlib.sha256(modified).hexdigest()}) + if not changed: + raise ValueError("SampleProject fixture has no blend materials for the negative control") + return changed + + +def compare_reviewed_reference(path, candidate, regions): + reference = read_ppm(path) + width, height = candidate[:2] + return {"reference_pixel_sha256": hashlib.sha256(reference[2]).hexdigest(), + "viewport": compare_region(reference, candidate, (0, 0, width, height)), + "regions": {name: compare_region(reference, candidate, rect) for name, rect in regions}} + + +def review_result(path, candidate, regions, review_identifier): + try: + result = compare_reviewed_reference(path, candidate, regions) + except ValueError as error: + # Keep the source/cooked comparison available when reference geometry + # or format does not match the requested profile. + result = {"error": str(error)} + result["review_identifier"] = review_identifier + return result + + +def count_dark_pixels(capture, rect, threshold=100): + width, height, pixels = capture + x, y, extent_x, extent_y = rect + if x < 0 or y < 0 or extent_x < 1 or extent_y < 1 or x + extent_x > width or y + extent_y > height: + raise ValueError("presence region is empty or outside the capture") + return sum(max(pixels[offset:offset + 3]) < threshold + for row in range(y, y + extent_y) + for offset in range((row * width + x) * 3, (row * width + x + extent_x) * 3, 3)) + + +def parse_presence(text): + try: + name, minimum = text.split(":", 1) + count = int(minimum) + except ValueError as error: + raise argparse.ArgumentTypeError("expected region-name:minimum-dark-pixels") from error + if not name or count < 1: + raise argparse.ArgumentTypeError("expected a region name and positive minimum pixel count") + return name, count + + +def validate_control_baseline(evidence, baseline, binary_hashes, shaders, device): + """Require a negative-control baseline to come from the same capture setup.""" + if device is None or evidence.get("device") != device or evidence.get("binary_sha256") != binary_hashes or evidence.get("shader_cache") != shaders: + raise ValueError("control baseline has different or incomplete build/device identity") + if evidence.get("source_pixel_sha256") != hashlib.sha256(baseline[2]).hexdigest(): + raise ValueError("control baseline pixels do not match their evidence") + + +def validation_layer_status(path): + return validation_layer_status_text(Path(path).read_text(encoding="utf-8")) + + +def run_capture(command, log_directory, timeout): + stdout_path, stderr_path = log_directory / "stdout.log", log_directory / "stderr.log" + with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + subprocess.run(command, check=True, timeout=timeout, stdout=stdout, stderr=stderr) + require_clean_vulkan_console(read_console_log(stdout_path)) + require_clean_vulkan_console(read_console_log(stderr_path)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project", type=Path, required=True) + parser.add_argument("--cooker", type=Path, required=True) + parser.add_argument("--obelisk", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--minimum-assets", type=int, default=32) + parser.add_argument("--expected-draws", type=int, default=16) + parser.add_argument("--timeout", type=float, default=180) + parser.add_argument("--region", action="append", type=parse_region, default=[], help="compare a named object region: name:x,y,width,height") + parser.add_argument("--presence", action="append", type=parse_presence, default=[], help="require dark pixels in a named region: region-name:minimum-count") + parser.add_argument("--reviewed-reference", type=Path, help="backend-specific reviewed RGB8 reference capture") + parser.add_argument("--reference-review", help="record the reference review identifier") + parser.add_argument("--require-validation", action="store_true", help="require Khronos validation in both capture processes") + parser.add_argument("--fixed-window", action="store_true", help="capture both processes at a fixed 1280x720 logical window") + parser.add_argument("--hierarchy", action="store_true", help="parent the SampleProject mesh while preserving its world transform in the copied scene") + parser.add_argument("--replacement", action="store_true", help="replace the flat scene with the hierarchy variant before capture") + parser.add_argument("--force-opaque", action="store_true", help="change copied blend materials to opaque for a visual negative control") + parser.add_argument("--no-directional-light", action="store_true", help="disable the copied scene's directional light for a visual negative control") + parser.add_argument("--different-from", type=Path, help="require the source and cooked captures to differ from this reference") + parser.add_argument("--minimum-different-pixels", type=int, default=0, help="minimum pixels differing by the comparison channel threshold") + parser.add_argument("--baseline-evidence", type=Path, help="verify the negative-control reference's binary, shader, device and pixel identities") + args = parser.parse_args() + project, cooker, obelisk, output = (p.resolve(strict=True) for p in (args.project, args.cooker, args.obelisk, args.output)) + if not project.is_file() or not output.is_dir() or any(output.iterdir()) or args.timeout <= 0 or args.minimum_assets < 1 or args.expected_draws < 1: + parser.error("provide a project file, an empty output directory, and positive limits") + if args.replacement and not args.hierarchy: + parser.error("--replacement requires --hierarchy") + if args.minimum_different_pixels < 0 or bool(args.different_from) != (args.minimum_different_pixels > 0): + parser.error("--different-from and a positive --minimum-different-pixels must be provided together") + if args.baseline_evidence and not args.different_from: + parser.error("--baseline-evidence requires --different-from") + if output == project.parent or project.parent in output.parents: + parser.error("output must be outside the project workspace") + if bool(args.reviewed_reference) != bool(args.reference_review): + parser.error("--reviewed-reference and --reference-review must be provided together") + if len({name for name, _ in args.region}) != len(args.region): + parser.error("region names must be unique") + if len({name for name, _ in args.presence}) != len(args.presence) or not {name for name, _ in args.presence}.issubset({name for name, _ in args.region}): + parser.error("presence checks must reference unique declared regions") + binary_hashes = {"cooker_sha256": sha256_file(cooker), "runtime_sha256": sha256_file(obelisk)} + asset_root = Path(os.environ.get("ZENGINE_ASSET_ROOT") or obelisk.parent / "ZodiacEngine").resolve(strict=True) + shader_cache = asset_root / "Shaders" / "Cache" + shaders = shader_identity(shader_cache) + for directory, names, files in os.walk(project.parent, followlinks=False): + if any((Path(directory) / name).is_symlink() for name in names + files): + raise ValueError("project copy contains a symlink") + source_scene = project.parent / "Scenes" / "level01.zescene" + original = source_scene.read_bytes() + if CAMERA.encode() in original or not original.endswith(b"\n"): + raise ValueError("scene cannot safely receive the fixture camera") + runtime_log = output / "runtime-log" + runtime_log.mkdir() + source_log = output / "source-log" + source_log.mkdir() + capture = output / "runtime.ppm" + source_capture = output / "source.ppm" + with tempfile.TemporaryDirectory(prefix="renderer-sample-runtime-") as temporary: + root = Path(temporary) + workspace = root / "project" + shutil.copytree(project.parent, workspace) + copied_project = workspace / project.name + config = json.loads(copied_project.read_text(encoding="utf-8")) + if config.get("workingSpace") != ".": + raise ValueError("project must use a project-local workspace") + modified_materials = force_opaque_materials(workspace / "Assets" / "Materials") if args.force_opaque else [] + config["sceneList"] = [{"name": "level01", "isDefault": True}] + if args.replacement: + config["sceneList"].append({"name": "level01_hierarchy", "isDefault": False}) + copied_project.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + variant = remove_sampleproject_directional_light(original) if args.no_directional_light else original + augmented = (add_sampleproject_hierarchy(variant) if args.hierarchy else variant) + CAMERA_ENTITY.encode() + scene_path = workspace / "Scenes" / "level01.zescene" + if args.replacement: + marker = ('"uuid": "' + SCENE + '"').encode() + if original.count(marker) != 1 or ALTERNATE_SCENE.encode() in original: + raise ValueError("SampleProject scene cannot safely receive the alternate identity") + alternate = augmented.replace(marker, ('"uuid": "' + ALTERNATE_SCENE + '"').encode(), 1) + scene_path.write_bytes(variant + CAMERA_ENTITY.encode()) + (workspace / "Scenes" / "level01_hierarchy.zescene").write_bytes(alternate) + else: + scene_path.write_bytes(augmented) + cook = root / "cook" + cook.mkdir() + subprocess.run([str(cooker), "smoke-offline", str(copied_project), str(cook), "805306368"], check=True, timeout=args.timeout) + package = cook / "output" / "desktop-vulkan-le64" + manifest_bytes = (package / "manifest.json").read_bytes() + manifest = json.loads(manifest_bytes) + selected = [item for item in manifest["scenes"] if Path(item["source"]).name == "level01.zescene"] + if len(selected) != 1 or len(manifest["assets"]) < args.minimum_assets: + raise ValueError("cooked manifest does not contain the required scene and assets") + if args.replacement: + if {item["uuid"] for item in manifest["scenes"]} != {SCENE, ALTERNATE_SCENE}: + raise ValueError("cooked manifest does not contain both replacement scenes") + scene_path.write_bytes(augmented) + fixed_window = ["--cookedFixedWindow"] if args.fixed_window else [] + run_capture([str(obelisk), "--launchEditor", "true", "--projectConfigFile", str(copied_project), "--sourceCapture", str(source_capture), "--cookedRuntimeView", "--cookedCamera", CAMERA, "--cookedExpectedDraws", str(args.expected_draws), "--logDirectory", str(source_log), *fixed_window], source_log, args.timeout) + source_text = read_process_logs(source_log) + validate_capture_log_text(source_text) + source_validation = validation_layer_status_text(source_text) + if args.require_validation and not source_validation: + raise ValueError("source capture did not enable Khronos Vulkan validation") + shutil.rmtree(workspace) + replacement = ["--cookedReloads", "1", "--cookedAlternateScene", ALTERNATE_SCENE] if args.replacement else [] + run_capture([str(obelisk), "--cookedStandalone", "--cookedPackage", str(package), "--cookedScene", selected[0]["uuid"], "--cookedCamera", CAMERA, "--cookedExitAfterFrame", "--cookedCapture", str(capture), "--cookedExpectedDraws", str(args.expected_draws), "--cookedAuditSource", "--logDirectory", str(runtime_log), *fixed_window, *replacement], runtime_log, args.timeout) + runtime_text = read_process_logs(runtime_log) + if args.replacement: + published = re.findall(r"\[CookedRuntime\] Published cooked scene generation (\d+) scene=([0-9a-f-]+)", runtime_text) + if published != [("2", SCENE), ("3", ALTERNATE_SCENE)]: + raise ValueError("replacement capture did not publish the expected scenes") + validate_capture_log_text(runtime_text) + runtime_validation = validation_layer_status_text(runtime_text) + if args.require_validation and not runtime_validation: + raise ValueError("runtime capture did not enable Khronos Vulkan validation") + require_zero_source_audit(runtime_text) + source_device, runtime_device = device_identity(source_text), device_identity(runtime_text) + if source_device != runtime_device: + raise ValueError("source and cooked captures used different or incompletely identified devices") + if sha256_file(cooker) != binary_hashes["cooker_sha256"] or sha256_file(obelisk) != binary_hashes["runtime_sha256"] or shader_identity(shader_cache) != shaders: + raise ValueError("binary or shader cache changed during capture") + width, height, pixels = read_ppm(capture) + if min(pixels) == max(pixels): + raise ValueError("runtime scene capture has no visible variation") + reference = read_ppm(source_capture) + candidate = (width, height, pixels) + viewport = compare_region(reference, candidate, (0, 0, width, height)) + regions = {name: compare_region(reference, candidate, rect) for name, rect in args.region} + rectangles = dict(args.region) + presence = {} + for name, minimum in args.presence: + source_count = count_dark_pixels(reference, rectangles[name]) + runtime_count = count_dark_pixels(candidate, rectangles[name]) + presence[name] = {"threshold_rgb_exclusive": 100, "minimum_pixels": minimum, + "source_pixels": source_count, "runtime_pixels": runtime_count, + "passed": source_count >= minimum and runtime_count >= minimum} + report = {"scope": "source-versus-standalone cooked SampleProject smoke; added camera; visual review required", + "original_scene_sha256": hashlib.sha256(original).hexdigest(), + "binary_sha256": binary_hashes, "shader_cache": shaders, "device": runtime_device, + "augmented_scene_sha256": hashlib.sha256(augmented).hexdigest(), + "manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(), + "scene_uuid": selected[0]["uuid"], "camera_uuid": CAMERA, + "captured_scene_uuid": ALTERNATE_SCENE if args.replacement else selected[0]["uuid"], + "alternate_scene_sha256": hashlib.sha256(alternate).hexdigest() if args.replacement else None, + "camera_position": [0.5, 2, 6], "cooked_assets": len(manifest["assets"]), + "source_audit": "zero denied instrumented engine-VFS source reads, registry lookups, and import enqueues from service initialization through joined-worker shutdown", + "expected_draws": args.expected_draws, "width": width, "height": height, + "source_pixel_sha256": hashlib.sha256(reference[2]).hexdigest(), + "runtime_pixel_sha256": hashlib.sha256(pixels).hexdigest(), + "fixed_window": args.fixed_window, + "hierarchy_fixture": args.hierarchy, + "replacement_fixture": args.replacement, + "opaque_negative_control": modified_materials, + "directional_light_disabled": args.no_directional_light, + "source_validation_enabled": source_validation, "runtime_validation_enabled": runtime_validation, + "viewport": viewport, "regions": regions, "presence": presence} + if args.different_from: + baseline = read_ppm(args.different_from) + if args.baseline_evidence: + validate_control_baseline(json.loads(args.baseline_evidence.read_text(encoding="utf-8")), baseline, binary_hashes, shaders, runtime_device) + source_difference = compare_region(baseline, reference, (0, 0, width, height)) + runtime_difference = compare_region(baseline, candidate, (0, 0, width, height)) + for difference in (source_difference, runtime_difference): + difference["within_similarity_tolerance"] = difference.pop("passed") + report["negative_control"] = {"baseline_pixel_sha256": hashlib.sha256(baseline[2]).hexdigest(), + "baseline_identity_verified": bool(args.baseline_evidence), + "minimum_different_pixels": args.minimum_different_pixels, + "source": source_difference, "runtime": runtime_difference, + "passed": min(source_difference["changed_pixels"], runtime_difference["changed_pixels"]) >= args.minimum_different_pixels} + if args.reviewed_reference: + report["reviewed_reference"] = review_result(args.reviewed_reference, candidate, args.region, args.reference_review) + (output / "runtime-smoke.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2)) + reviewed = report.get("reviewed_reference") + reviewed_passed = not reviewed or ("error" not in reviewed and reviewed["viewport"]["passed"] and all(item["passed"] for item in reviewed["regions"].values())) + if not viewport["passed"] or not all(item["passed"] for item in regions.values()) or not all(item["passed"] for item in presence.values()) or not reviewed_passed or not report.get("negative_control", {"passed": True})["passed"]: + return 1 + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError, KeyError, subprocess.SubprocessError) as error: + print(f"SampleProject runtime smoke failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/tests/scene/sampleproject_cooked_timing.py b/tests/scene/sampleproject_cooked_timing.py new file mode 100644 index 000000000..291fb6427 --- /dev/null +++ b/tests/scene/sampleproject_cooked_timing.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +"""Measure copied SampleProject loads and replacements; not qualification. + +The fixture adds one camera and duplicates level01 with a distinct scene UUID. +The two scenes share assets. The authoring copy is removed before runtime. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import shutil +import subprocess +import sys +import tempfile + +from analyze_cooked_timing_samples import analyze +from analyze_cooked_runtime_trace import analyze as analyze_trace +from cooked_capture_identity import device_identity, shader_identity +from cooked_scene_smoke import (observed_timing_viewport, read_lifecycle_observations, read_process_logs, + require_clean_stress_log, require_zero_source_audit, sha256_file, + validation_layer_status_text) +from sampleproject_cooked_runtime import CAMERA, CAMERA_ENTITY, run_capture + + +SCENE = "f6d4dbea-c21d-453d-ae91-79187dfa8b84" +ALTERNATE = "f6d4dbea-c21d-453d-ae91-79187dfa8b85" + + +def host_hardware(): + if platform.system() != "Darwin": + return None + try: + cpu = subprocess.check_output(["sysctl", "-n", "machdep.cpu.brand_string"], text=True, timeout=5).strip() + memory = int(subprocess.check_output(["sysctl", "-n", "hw.memsize"], text=True, timeout=5).strip()) + except (OSError, ValueError, subprocess.SubprocessError): + return None + return {"cpu": cpu, "physical_memory_bytes": memory} if cpu and memory > 0 else None + + +def memory_summary(samples): + if not samples: + raise ValueError("memory summary requires lifecycle observations") + fields = {"cpu_pool_peak_bytes": "cpu_peak", "preparation_staging_peak_bytes": "staging_peak", + "tracked_gpu_allocation_peak_bytes": "gpu_peak", "metadata_max_live_bytes": "metadata_live", + "metadata_max_live_blocks": "metadata_blocks", "retirement_max_queued_at_observation": "retirement_queued", + "retirement_max_draining_at_observation": "retirement_draining"} + if any("gpu_admitted_peak" in sample for sample in samples) and not all("gpu_admitted_peak" in sample for sample in samples): + raise ValueError("partial GPU budget observations") + if all("gpu_admitted_peak" in sample for sample in samples): + fields.update({"gpu_admitted_peak_bytes": "gpu_admitted_peak", "gpu_geometry_peak_bytes": "gpu_geometry_peak", + "gpu_texture_peak_bytes": "gpu_texture_peak", "gpu_target_peak_bytes": "gpu_target_peak", + "gpu_uniform_peak_bytes": "gpu_uniform_peak", "gpu_staging_peak_bytes": "gpu_staging_peak", + "gpu_readback_peak_bytes": "gpu_readback_peak"}) + if any("retirement_high_water" in sample for sample in samples) and not all("retirement_high_water" in sample for sample in samples): + raise ValueError("partial retirement budget observations") + if all("retirement_high_water" in sample for sample in samples): + fields["retirement_high_water"] = "retirement_high_water" + result = {} + for label, key in fields.items(): + values = [sample[key] for sample in samples] + if any(isinstance(value, bool) or not isinstance(value, int) or value < 0 for value in values): + raise ValueError("invalid lifecycle memory observation") + result[label] = max(values) + if "retirement_high_water" in fields: + capacities = {sample["retirement_capacity"] for sample in samples} + if len(capacities) != 1 or next(iter(capacities)) <= 0: + raise ValueError("retirement capacity changed across observations") + result["retirement_capacity"] = next(iter(capacities)) + result["scope"] = "tracked allocator peaks and sampled metadata/retirement counts; excludes driver residency" + return result + + +def require_visible_trace(report): + windows = report.get("windows", {}) + required = {"initial_load", "steady", "replacement", "post_steady"} + if set(windows) != required: + raise ValueError("visible trace requires all four timing windows") + for name, window in windows.items(): + state = window.get("window_occlusion") + if not state or state["visible_at_start"] != 1 or any(change["visible"] != 1 for change in state["changes"]): + raise ValueError(f"window visibility is not proven throughout {name}; inspect the trace") + + +def require_visible_samples(log, samples): + matches = re.findall(r"\[CookedWindowVisibility\] timestamp_ns=(\d+) visible=([012])", log) + if not matches or log.count("[CookedWindowVisibility]") != len(matches): + raise ValueError("missing or malformed per-process visibility observations") + events = [(int(timestamp), int(visible)) for timestamp, visible in matches] + if any(timestamp <= 0 for timestamp, _ in events) or any(events[index][0] <= events[index - 1][0] for index in range(1, len(events))): + raise ValueError("visibility observations are not ordered") + for sample in samples: + start, end = sample["admission_ns"], sample["render_completion_observed_ns"] + prior = [visible for timestamp, visible in events if timestamp <= start] + if not prior or prior[-1] != 1 or any(visible != 1 for timestamp, visible in events if start < timestamp <= end): + raise ValueError("window visibility is not proven throughout a load sample") + sample["visible_window_verified"] = True + + +def scene_pair(original): + marker = ('"uuid": "' + SCENE + '"').encode() + if original.count(marker) != 1 or CAMERA.encode() in original or not original.endswith(b"\n"): + raise ValueError("SampleProject scene cannot safely receive the fixture camera and alternate UUID") + initial = original + CAMERA_ENTITY.encode() + alternate = initial.replace(marker, ('"uuid": "' + ALTERNATE + '"').encode(), 1) + return initial, alternate + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project", type=Path, required=True) + parser.add_argument("--cooker", type=Path, required=True) + parser.add_argument("--obelisk", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--count", type=int, default=30) + parser.add_argument("--expected-draws", type=int, default=16) + parser.add_argument("--timeout", type=float, default=600) + parser.add_argument("--trace-steady-frames", type=int, default=0, help="also capture steady frames before and after one replacement") + parser.add_argument("--require-visible-trace", action="store_true", help="reject a trace without proven window visibility throughout its timing windows") + args = parser.parse_args() + project, cooker, obelisk, output = (path.resolve(strict=True) for path in (args.project, args.cooker, args.obelisk, args.output)) + if not project.is_file() or not output.is_dir() or any(output.iterdir()) or not 1 <= args.count <= 100 or args.expected_draws < 1 or args.timeout <= 0 or (args.trace_steady_frames and not 2 <= args.trace_steady_frames <= 600) or (args.require_visible_trace and not args.trace_steady_frames): + parser.error("provide a project, empty output directory and positive bounded limits") + if output == project.parent or project.parent in output.parents: + parser.error("output must be outside the project workspace") + for directory, names, files in os.walk(project.parent, followlinks=False): + if any((Path(directory) / name).is_symlink() for name in names + files): + raise ValueError("project copy contains a symlink") + original = (project.parent / "Scenes" / "level01.zescene").read_bytes() + initial, alternate = scene_pair(original) + hashes = {"cooker_sha256": sha256_file(cooker), "runtime_sha256": sha256_file(obelisk)} + asset_root = Path(os.environ.get("ZENGINE_ASSET_ROOT") or obelisk.parent / "ZodiacEngine").resolve(strict=True) + shader_cache = asset_root / "Shaders" / "Cache" + shaders = shader_identity(shader_cache) + hardware = host_hardware() + with tempfile.TemporaryDirectory(prefix="renderer-sample-timing-") as temporary: + root = Path(temporary) + workspace = root / "project" + shutil.copytree(project.parent, workspace) + config_path = workspace / project.name + config = json.loads(config_path.read_text(encoding="utf-8")) + if config.get("workingSpace") != ".": + raise ValueError("project must use a project-local workspace") + config["sceneList"] = [{"name": "level01", "isDefault": True}, {"name": "level01_timing_alternate", "isDefault": False}] + config_path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + (workspace / "Scenes" / "level01.zescene").write_bytes(initial) + (workspace / "Scenes" / "level01_timing_alternate.zescene").write_bytes(alternate) + cook = root / "cook" + cook.mkdir() + subprocess.run([str(cooker), "smoke-offline", str(config_path), str(cook), "805306368"], check=True, timeout=args.timeout) + package = cook / "output" / "desktop-vulkan-le64" + manifest_bytes = (package / "manifest.json").read_bytes() + manifest = json.loads(manifest_bytes) + if {item["uuid"] for item in manifest["scenes"]} != {SCENE, ALTERNATE} or len(manifest["assets"]) < 32: + raise ValueError("cooked package lacks the expected scenes or assets") + shutil.rmtree(workspace) + common = {"manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(), "binary_sha256": hashes, + "host_system": platform.platform(), "host_machine": platform.machine(), + "shader_cache": shaders, "hardware": hardware, + "filesystem_cache_conditions": "uncontrolled operating-system cache; no cold-disk claim", + "source_audit_enabled": True, "gpu_completion_observed": True} + + devices = [] + + def capture(name, reloads, steady_frames=0, trace=None): + if shader_identity(shader_cache) != shaders: + raise ValueError("runtime shader cache changed before a timing process") + logs = output / name + logs.mkdir() + command = [str(obelisk), "--cookedStandalone", "--cookedPackage", str(package), "--cookedScene", SCENE, + "--cookedCamera", CAMERA, "--cookedExitAfterFrame", "--cookedExpectedDraws", str(args.expected_draws), + "--cookedAuditSource", "--cookedObserveGpuCompletion", "--cookedFixedWindow", "--logDirectory", str(logs)] + if reloads: + command.extend(["--cookedReloads", str(reloads), "--cookedAlternateScene", ALTERNATE]) + if args.require_visible_trace: + command.append("--cookedWaitVisible") + if steady_frames: + command.extend(["--cookedSteadyFrames", str(steady_frames), "--cookedPostSteadyFrames", str(steady_frames), "--cookedStageTrace", str(trace)]) + run_capture(command, logs, args.timeout) + if shader_identity(shader_cache) != shaders: + raise ValueError("runtime shader cache changed during a timing process") + log = read_process_logs(logs) + devices.append(device_identity(log)) + require_zero_source_audit(log) + require_clean_stress_log(log) + if not validation_layer_status_text(log) or "Engine destroyed" not in log: + raise ValueError("runtime validation or clean shutdown was not confirmed") + expected = [(generation, SCENE if generation % 2 == 0 else ALTERNATE) for generation in range(2, reloads + 3)] + observed = [(int(generation), scene) for generation, scene in re.findall(r"\[CookedRuntime\] Published cooked scene generation (\d+) scene=([0-9a-f-]+)", log)] + if observed != expected or f"[CookedRuntime] Submitted {args.expected_draws} draws" not in log: + raise ValueError("runtime did not publish and render the expected SampleProject scenes") + if sha256_file(obelisk) != hashes["runtime_sha256"]: + raise ValueError("runtime binary changed during timing collection") + samples = read_lifecycle_observations(log, [generation for generation, _ in expected], True, require_gpu_budget=True) + if args.require_visible_trace: + require_visible_samples(log, samples) + if steady_frames: + windows = [tuple(map(int, row)) for row in re.findall(r"\[CookedSteady\] generation=(\d+) start_ns=(\d+) end_ns=(\d+) submitted_frames=(\d+)", log)] + if len(windows) != 1: + raise ValueError("missing or repeated steady-rendering window") + generation, start, end, frames = windows[0] + if generation != 2 or frames < steady_frames or start < samples[0]["first_submission_ns"] or end <= start or end > samples[1]["admission_ns"]: + raise ValueError("invalid steady-rendering window") + samples[0].update(steady_start_ns=start, steady_end_ns=end, steady_submitted_frames=frames) + post = [tuple(map(int, row)) for row in re.findall(r"\[CookedPostSteady\] generation=(\d+) start_ns=(\d+) end_ns=(\d+) submitted_frames=(\d+)", log)] + if len(post) != 1: + raise ValueError("missing or repeated post-replacement steady window") + generation, start, end, frames = post[0] + if generation != 3 or frames < steady_frames or start < samples[1]["first_submission_ns"] or end <= start: + raise ValueError("invalid post-replacement steady window") + samples[1].update(post_steady_start_ns=start, post_steady_end_ns=end, post_steady_submitted_frames=frames) + return samples, observed_timing_viewport(log) + + fresh_samples, viewports = [], [] + for index in range(args.count): + samples, viewport = capture(f"fresh-{index + 1:02d}", 0) + fresh_samples.extend(samples) + viewports.append(viewport) + samples, viewport = capture("replacements", args.count) + if sha256_file(cooker) != hashes["cooker_sha256"]: + raise ValueError("cooker binary changed during timing collection") + fresh = {**common, "fresh_process_samples": fresh_samples, "validation_states": [True] * args.count, "timing_viewports": viewports} + replacements = {**common, "reloads": args.count, "samples": samples, "validation_states": [True], "timing_viewports": [viewport]} + fresh["device"] = devices[0] + replacements["device"] = devices[0] + report = analyze(fresh, replacements, args.count) + trace_report = None + if args.trace_steady_frames: + trace_path = output / "stage.csv" + trace_samples, trace_viewport = capture("trace", 1, args.trace_steady_frames, trace_path) + if trace_viewport != viewport or trace_viewport is None or trace_viewport["resize_after_baseline"]: + raise ValueError("trace viewport differs from timing samples") + trace_report = analyze_trace({**common, "samples": trace_samples}, trace_path, args.trace_steady_frames) + if shader_identity(shader_cache) != shaders: + raise ValueError("runtime shader cache changed during timing collection") + if any(device != devices[0] for device in devices[1:]): + raise ValueError("GPU or driver identity changed during timing collection") + if trace_report is not None: + trace_report.update(binary_sha256=hashes, shader_cache=shaders, hardware=hardware, device=devices[0], viewport=viewport) + (output / "trace.json").write_text(json.dumps(trace_report, indent=2) + "\n", encoding="utf-8") + if args.require_visible_trace: + require_visible_trace(trace_report) + report.update(fixture="SampleProject level01 copy with added camera and same-assets alternate scene", + shader_cache=shaders, + hardware=hardware, device=devices[0], + memory={"fresh_processes": memory_summary(fresh_samples), + "same_process_replacements": memory_summary(samples)}, + original_scene_sha256=hashlib.sha256(original).hexdigest(), + augmented_scene_sha256=hashlib.sha256(initial).hexdigest(), + alternate_scene_sha256=hashlib.sha256(alternate).hexdigest(), cooked_assets=len(manifest["assets"])) + (output / "fresh.json").write_text(json.dumps(fresh, indent=2) + "\n", encoding="utf-8") + (output / "replacements.json").write_text(json.dumps(replacements, indent=2) + "\n", encoding="utf-8") + (output / "timing.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as error: + print(f"SampleProject timing failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/tests/scene/unit/test_cooked_capacity_probe.py b/tests/scene/unit/test_cooked_capacity_probe.py new file mode 100644 index 000000000..a6b298103 --- /dev/null +++ b/tests/scene/unit/test_cooked_capacity_probe.py @@ -0,0 +1,107 @@ +from pathlib import Path +import re +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).parents[1])) +from cooked_capacity_probe import ALTERNATE_MATERIAL, ALTERNATE_SCENE, MATERIAL, MAX_DRAWS, SCENE, append_instances, make_material_variants, make_texture_variants, summarize_initial_trace, validate_publications, validate_unique_material_manifest + + +class CookedCapacityFixtureTest(unittest.TestCase): + def test_texture_variants_preserve_layout_and_have_distinct_pixels_and_ids(self): + import json + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + pixels = bytearray(58) + pixels[:2] = b"BM" + pixels[10:14] = (54).to_bytes(4, "little") + pixels[54:57] = b"\x0a\x14\x1e" + (root / "pixel.bmp").write_bytes(pixels) + (root / "pixel.bmp.meta").write_text(json.dumps({"uuid": "50000000-0000-0000-0000-000000000005"})) + identities = make_texture_variants(root, 3) + self.assertEqual(len({identity for identity, _ in identities}), 3) + images = [(root / Path(path).name).read_bytes() for _, path in identities] + self.assertEqual(len(set(images)), 3) + self.assertTrue(all(image[:54] == images[0][:54] for image in images)) + + def test_material_variants_have_distinct_identities_and_scene_references(self): + import json + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "material.zematerial").write_text(json.dumps({"uuid": MATERIAL, "name": "fixture"})) + (root / "material.zematerial.meta").write_text(json.dumps({"uuid": MATERIAL, "importer": "MaterialImporter"})) + identities = make_material_variants(root, 3) + self.assertEqual(len(set(identities)), 3) + self.assertEqual(json.loads((root / "capacity-material-00001.zematerial").read_text())["uuid"], identities[1]) + scene = root / "scene" + scene.write_text('"entities":\n') + append_instances(scene, 4, identities) + self.assertIn(identities[1], scene.read_text()) + self.assertIn(identities[2], scene.read_text()) + + def test_distinct_material_roots_and_objects_are_required(self): + manifest = {"scenes": [{"uuid": SCENE, "roots": [{"kind": "material", "uuid": MATERIAL}]}, + {"uuid": ALTERNATE_SCENE, "roots": [{"kind": "material", "uuid": ALTERNATE_MATERIAL}]}], + "assets": [{"kind": "material", "uuid": MATERIAL, "object": {"digest": "a"}}, + {"kind": "material", "uuid": ALTERNATE_MATERIAL, "object": {"digest": "b"}}]} + validate_unique_material_manifest(manifest) + manifest["assets"][1]["object"]["digest"] = "a" + with self.assertRaisesRegex(ValueError, "distinct cooked objects"): + validate_unique_material_manifest(manifest) + + def test_trace_reports_partial_admission_coverage_and_rejects_truncation(self): + with tempfile.TemporaryDirectory() as directory: + trace = Path(directory) / "stage.csv" + trace.write_text("kind,name,timestamp_ns,value\nvalue,SceneLoad.Main.FrameWorkMs,110,0.00001\nscope_ns,SceneLoad.Main.RuntimeUpdate,109,3\nvalue,SceneLoad.Main.FrameWorkMs,210,0.00001\n", encoding="utf-8") + sample = {"admission_ns": 100, "first_submission_ns": 200} + report = summarize_initial_trace(trace, sample) + self.assertEqual(report["starts_after_admission_ns"], 9) + self.assertFalse(report["admission_marker_observed"]) + self.assertEqual(report["stages"]["SceneLoad.Main.FrameWorkMs"]["count"], 1) + self.assertAlmostEqual(report["main_frame_diagnostics"]["runtime_update_ms"]["max_ms"], 0.000003) + with self.assertRaisesRegex(ValueError, "does not cover"): + summarize_initial_trace(trace, {"admission_ns": 100, "first_submission_ns": 220}) + with trace.open("a", encoding="utf-8") as output: + output.write("scope_ns,SceneLoad.Main.BeforeInitialAdmission,99,1\n") + self.assertTrue(summarize_initial_trace(trace, sample)["admission_marker_observed"]) + + def test_refusal_rejects_partial_or_stale_publications_and_fallback(self): + initial = f"[CookedRuntime] Published cooked scene generation 2 scene={SCENE}\n" + retry = f"[CookedRuntime] Published cooked scene generation 3 scene={SCENE}\n" + self.assertEqual(validate_publications(initial + retry, False, True), [2, 3]) + self.assertEqual(validate_publications(initial + retry.replace(SCENE, ALTERNATE_SCENE), False, False, True), [2, 3]) + with self.assertRaisesRegex(ValueError, "unexpected scene"): + validate_publications(initial + retry, False, False, True) + with self.assertRaisesRegex(ValueError, "unexpected scene"): + validate_publications(initial + retry.replace(SCENE, "90000000-0000-0000-0000-000000000009"), False, True) + with self.assertRaisesRegex(ValueError, "unexpected scene"): + validate_publications(initial, True, False) + with self.assertRaisesRegex(ValueError, "environment fallback"): + validate_publications(initial + "[SkyEnvironment] Revision 4 is using the fallback\n" + retry, False, True) + + def test_appends_distinct_instances_without_changing_original_scene(self): + with tempfile.TemporaryDirectory() as directory: + scene = Path(directory) / "fixture.scene" + original = b'"entities":\n - "uuid": "original"\n' + scene.write_bytes(original) + append_instances(scene, 4) + data = scene.read_bytes() + self.assertTrue(data.startswith(original)) + identities = re.findall(rb'"uuid": "(d[0-9a-f-]+)"', data) + self.assertEqual(len(identities), 3) + self.assertEqual(len(set(identities)), 3) + self.assertEqual(data.count(b'"MeshComponent"'), 3) + + def test_rejects_out_of_range_and_incomplete_scene_without_mutation(self): + with tempfile.TemporaryDirectory() as directory: + scene = Path(directory) / "fixture.scene" + scene.write_bytes(b"incomplete") + for count in (0, MAX_DRAWS + 2, 1): + with self.assertRaises(ValueError): + append_instances(scene, count) + self.assertEqual(scene.read_bytes(), b"incomplete") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/scene/unit/test_cooked_runtime_trace.py b/tests/scene/unit/test_cooked_runtime_trace.py new file mode 100644 index 000000000..338445477 --- /dev/null +++ b/tests/scene/unit/test_cooked_runtime_trace.py @@ -0,0 +1,141 @@ +import csv +from pathlib import Path +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).parents[1])) +from analyze_cooked_runtime_trace import analyze, render_frame_diagnostics, render_outliers, window_occlusion + + +class RuntimeTraceTest(unittest.TestCase): + def test_occlusion_carries_prior_state_and_records_transitions(self): + records = [("value", "SceneLoad.Main.WindowOcclusionVisible", 10, 1), + ("value", "SceneLoad.Main.WindowOcclusionVisible", 30, 0), + ("value", "SceneLoad.Main.WindowOcclusionVisible", 50, 1)] + result = window_occlusion(records, (20, 40)) + self.assertEqual(result["visible_at_start"], 1) + self.assertEqual(result["changes"], [{"timestamp_ns": 30, "visible": 0}]) + self.assertIsNone(window_occlusion(records, (0, 5))) + with self.assertRaisesRegex(ValueError, "invalid window"): + window_occlusion(records + [("value", "SceneLoad.Main.WindowOcclusionVisible", 35, 3)], (20, 40)) + + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.trace = Path(self.temp.name) / "trace.csv" + self.evidence = {"manifest_sha256": "fixture", "filesystem_cache_conditions": "uncontrolled", + "samples": [{"generation": 2, "admission_ns": 100, "first_submission_ns": 200, + "steady_start_ns": 210, "steady_end_ns": 300, "steady_submitted_frames": 2}, + {"generation": 3, "admission_ns": 310, "first_submission_ns": 400}]} + self.rows = [] + for timestamp in (110, 220, 280, 320, 390, 410): + for name in ("SceneLoad.Main.FrameWorkMs", "SceneLoad.Main.FrameIntervalMs", "SceneLoad.Render.FrameIntervalMs"): + self.rows.append({"kind": "value", "name": name, "timestamp_ns": timestamp, "value": 1}) + self.write() + + def write(self): + with self.trace.open("w", newline="", encoding="utf-8") as output: + writer = csv.DictWriter(output, fieldnames=("kind", "name", "timestamp_ns", "value")) + writer.writeheader() + writer.writerows(self.rows) + + def test_reports_partial_initial_coverage(self): + report = analyze(self.evidence, self.trace, 2) + self.assertEqual(report["initial_trace_starts_after_admission_ns"], 10) + self.assertFalse(report["initial_admission_marker_observed"]) + self.assertEqual(report["windows"]["steady"]["stages"]["SceneLoad.Render.FrameIntervalMs"]["count"], 2) + + def test_post_replacement_window_requires_complete_ordered_evidence(self): + self.evidence["samples"][1].update(post_steady_start_ns=405, post_steady_end_ns=500, post_steady_submitted_frames=2) + for timestamp in (450, 480, 510): + for name in ("SceneLoad.Main.FrameWorkMs", "SceneLoad.Main.FrameIntervalMs", "SceneLoad.Render.FrameIntervalMs"): + self.rows.append({"kind": "value", "name": name, "timestamp_ns": timestamp, "value": 1}) + self.write() + self.assertIn("post_steady", analyze(self.evidence, self.trace, 2)["windows"]) + self.evidence["samples"][1]["post_steady_start_ns"] = 399 + with self.assertRaisesRegex(ValueError, "invalid post-replacement"): + analyze(self.evidence, self.trace, 2) + + def test_detects_a_pre_admission_trace_marker(self): + self.rows.append({"kind": "scope_ns", "name": "SceneLoad.Main.BeforeInitialAdmission", "timestamp_ns": 99, "value": 1}) + self.write() + report = analyze(self.evidence, self.trace, 2) + self.assertTrue(report["initial_admission_marker_observed"]) + self.assertEqual(report["initial_trace_starts_after_admission_ns"], 0) + + def test_reports_queue_submission_in_each_window(self): + for timestamp, duration in ((150, 20), (250, 10), (350, 30)): + self.rows.append({"kind": "scope_ns", "name": "SceneLoad.Render.QueueSubmit", + "timestamp_ns": timestamp, "value": duration}) + self.write() + windows = analyze(self.evidence, self.trace, 2)["windows"] + for name, duration in (("initial_load", 20), ("steady", 10), ("replacement", 30)): + self.assertAlmostEqual(windows[name]["stages"]["SceneLoad.Render.QueueSubmit"]["max_ms"], duration / 1_000_000) + + def test_rejects_overlapping_windows_and_incomplete_trace(self): + self.evidence["samples"][0]["steady_end_ns"] = 311 + with self.assertRaisesRegex(ValueError, "overlapping"): + analyze(self.evidence, self.trace, 2) + self.evidence["samples"][0]["steady_end_ns"] = 300 + self.rows = [row for row in self.rows if row["timestamp_ns"] < 400] + self.write() + with self.assertRaisesRegex(ValueError, "does not cover"): + analyze(self.evidence, self.trace, 2) + + def test_reports_contained_scopes_and_rejects_partial_attribution(self): + self.rows.extend([ + {"kind": "scope_ns", "name": "SceneLoad.Main.RuntimeUpdate", "timestamp_ns": 270, "value": 10}, + {"kind": "scope_ns", "name": "SceneLoad.Main.RuntimeUpdate", "timestamp_ns": 276, "value": 4}, + {"kind": "scope_ns", "name": "SceneLoad.Main.RuntimeGuardedCommit", "timestamp_ns": 268, "value": 3}, + {"kind": "scope_ns", "name": "SceneLoad.Main.RuntimeUpdate", "timestamp_ns": 290, "value": 5}, + ]) + # Synthetic frames must not overlap across the window boundaries. + for row in self.rows: + if row["name"] == "SceneLoad.Main.FrameWorkMs": + row["value"] = 0.00005 if row["timestamp_ns"] in (220, 280) else 0.00001 + self.write() + diagnostics = analyze(self.evidence, self.trace, 2)["windows"]["steady"]["main_frame_diagnostics"] + self.assertEqual(diagnostics["runtime_update_ms"]["count"], 1) + self.assertAlmostEqual(diagnostics["runtime_update_ms"]["max_ms"], 0.000014) + self.assertEqual(diagnostics["runtime_update_ms"]["worst_frame_end_ns"], 280) + self.assertEqual(diagnostics["guarded_commit_ms"]["count"], 1) + self.assertEqual(diagnostics["partial_scopes"], 1) + self.assertEqual(diagnostics["unassigned_scopes"][0]["end_ns"], 290) + self.assertAlmostEqual(diagnostics["unassigned_scopes"][0]["duration_ms"], 0.000005) + + def test_render_repeated_visits_are_summed_per_frame(self): + records = [("value", "SceneLoad.Render.FrameWorkMs", 100, 0.00005), + ("scope_ns", "SceneLoad.Render.EnvironmentPreparation", 80, 10), + ("scope_ns", "SceneLoad.Render.EnvironmentPreparation", 95, 5), + ("scope_ns", "SceneLoad.Render.CookedUploadVisit", 90, 3)] + result = render_frame_diagnostics(records, (50, 100)) + self.assertAlmostEqual(result["environment_preparation_ms"]["max_ms"], 0.000015) + self.assertEqual(result["environment_preparation_ms"]["count"], 1) + crossing = render_frame_diagnostics(records, (50, 96)) + self.assertEqual(crossing["partial_scopes"], 0) + self.assertAlmostEqual(crossing["environment_preparation_ms"]["max_ms"], 0.000015) + self.assertAlmostEqual(result["cooked_upload_ms"]["max_ms"], 0.000003) + + def test_render_outliers_report_overlaps_without_claiming_causation(self): + records = [("value", "SceneLoad.Render.FrameIntervalMs", 20_000_000, 12), + ("scope_ns", "SceneLoad.Render.EnvironmentPreparation", 19_000_000, 2_000_000), + ("scope_ns", "SceneLoad.Main.PlatformEvents", 18_000_000, 3_000_000), + ("value", "SceneLoad.RecordingWorker.DispatchDelayMs", 17_000_000, 6), + ("value", "SceneLoad.QueueSubmit.LongNonCpuMs", 19_500_000, 9), + ("value", "SceneLoad.Main.WindowOcclusionVisible", 7_000_000, 1), + ("value", "SceneLoad.Main.WindowOcclusionVisible", 16_000_000, 0), + ("scope_ns", "SceneLoad.Render.CookedUploadVisit", 30_000_000, 1_000_000)] + report = render_outliers(records, (1, 25_000_000), steady_p95_ms=3) + self.assertEqual(report["count"], 1) + self.assertEqual(report["largest_intervals"][0]["longest_overlapping_scopes"][0]["name"], "SceneLoad.Render.EnvironmentPreparation") + self.assertEqual(len(report["largest_intervals"][0]["longest_overlapping_scopes"]), 1) + self.assertEqual(report["largest_intervals"][0]["overlapping_platform_events"][0]["overlap_ms"], 3) + self.assertEqual(report["largest_intervals"][0]["recording_worker_samples"][0]["value_ms"], 6) + self.assertEqual(report["largest_intervals"][0]["queue_submit_samples"][0]["value_ms"], 9) + self.assertEqual(report["largest_intervals"][0]["window_occlusion"]["visible_at_start"], 1) + self.assertEqual(report["largest_intervals"][0]["window_occlusion"]["changes"][0]["visible"], 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/scene/unit/test_cooked_scene_parity.py b/tests/scene/unit/test_cooked_scene_parity.py new file mode 100644 index 000000000..21ad90938 --- /dev/null +++ b/tests/scene/unit/test_cooked_scene_parity.py @@ -0,0 +1,21 @@ +import sys +from pathlib import Path +import unittest + +sys.path.insert(0, str(Path(__file__).parents[1])) +from cooked_scene_parity import compare_transparency + + +class CookedSceneParityTest(unittest.TestCase): + def test_transparency_requires_intermediate_pixel_over_opaque_triangle(self): + opaque = (2, 1, bytes([200, 200, 200, 0, 0, 0])) + blended = (2, 1, bytes([200, 200, 200, 100, 100, 100])) + self.assertTrue(compare_transparency(opaque, blended)["passed"]) + self.assertFalse(compare_transparency(opaque, opaque)["passed"]) + self.assertFalse(compare_transparency(opaque, (2, 1, bytes([200] * 6)))["passed"]) + with self.assertRaisesRegex(ValueError, "dimensions"): + compare_transparency(opaque, (1, 1, bytes([100] * 3))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/scene/unit/test_cooked_smoke_logs.py b/tests/scene/unit/test_cooked_smoke_logs.py new file mode 100644 index 000000000..c423b2add --- /dev/null +++ b/tests/scene/unit/test_cooked_smoke_logs.py @@ -0,0 +1,158 @@ +from pathlib import Path +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).parents[1])) +from cooked_scene_smoke import ALTERNATE_SCENE, SCENE, observed_timing_viewport, read_cancellation_timings, read_gpu_cancellation_handoffs, read_lifecycle_observations, read_process_logs, require_clean_stress_log, require_clean_vulkan_console, require_stable_stress_window, require_zero_source_audit, sha256_file, validate_retry_publications, validate_standalone_package_failure, validate_stress_sequence + + +class CookedSmokeLogTest(unittest.TestCase): + def test_gpu_handoff_requires_complete_ordered_timing(self): + samples = [{"boundary": "gpu", "cancelled": 2, "request_ns": 10, "cleanup_ns": 50}] + marker = "[CookedGpuCancelTiming] request_ns=12 observed_ns=20 finished_ns=35\n" + self.assertEqual(read_gpu_cancellation_handoffs(marker, samples)[0]["observed_to_finished_ns"], 15) + self.assertEqual(read_gpu_cancellation_handoffs("", samples), []) + for bad in (marker + marker, marker.replace("observed_ns=20", "observed_ns=51"), "[CookedGpuCancelTiming] bad\n"): + with self.subTest(log=bad), self.assertRaises(ValueError): + read_gpu_cancellation_handoffs(bad, samples) + + def test_standalone_package_failure_rejects_crash_or_publication(self): + log = "[CookedRuntime] cooked reconstruction failed\nEngine destroyed\n" + validate_standalone_package_failure(2, log) + for code, bad in ((0, log), (2, "[CookedRuntime] unrelated crash\nEngine destroyed\n"), + (2, log.replace("Engine destroyed", "")), + (2, log + "[CookedRuntime] Published cooked scene generation 2\n")): + with self.subTest(code=code, log=bad), self.assertRaises(RuntimeError): + validate_standalone_package_failure(code, bad) + + def test_gpu_budget_observations_require_order_and_respect_domain_limits(self): + log = ("[CookedLifecycle] generation=2 cpu_live=1 cpu_peak=1 staging_live=1 staging_peak=1 " + "metadata_live=1 metadata_blocks=1 gpu_live=1 gpu_peak=1 retirement_queued=0 retirement_draining=0\n" + "[CookedCache] generation=2 closures=0 leases=0 retiring=0 asset_versions=0\n" + "[CookedTiming] generation=2 admission_ns=10 first_submission_ns=20 elapsed_ns=10\n") + with self.assertRaisesRegex(RuntimeError, "GPU budget"): + read_lifecycle_observations(log, [2], require_gpu_budget=True) + budget = "[CookedGpuBudget] generation=2 admitted_peak=100 geometry_peak=30 texture_peak=20 target_peak=10 uniform_peak=5 staging_peak=4 readback_peak=1\n" + retirement = "[CookedRetirementBudget] generation=2 capacity=20 high_water=10\n" + sample = read_lifecycle_observations(log + budget + retirement, [2], require_gpu_budget=True)[0] + self.assertEqual(sample["gpu_admitted_peak"], 100) + self.assertEqual(sample["gpu_geometry_peak"], 30) + self.assertEqual(sample["retirement_high_water"], 10) + with self.assertRaisesRegex(RuntimeError, "exceeds"): + read_lifecycle_observations(log + budget.replace("staging_peak=4", "staging_peak=536870913") + retirement, [2], require_gpu_budget=True) + with self.assertRaisesRegex(RuntimeError, "retirement high-water"): + read_lifecycle_observations(log + budget + retirement.replace("high_water=10", "high_water=21"), [2], require_gpu_budget=True) + + def test_cancellation_timings_require_order_boundaries_and_exact_durations(self): + log = ("[CookedCancelTiming] cancelled=1 boundary=cpu request_ns=10 cleanup_ns=20 elapsed_ns=10\n" + "[CookedCancelTiming] cancelled=2 boundary=gpu request_ns=30 cleanup_ns=45 elapsed_ns=15\n") + self.assertEqual([sample["elapsed_ns"] for sample in read_cancellation_timings(log, 4)], [10, 15]) + for bad in (log.replace("boundary=gpu", "boundary=cpu"), log.replace("elapsed_ns=15", "elapsed_ns=14"), log.splitlines()[0]): + with self.subTest(bad=bad), self.assertRaises(ValueError): + read_cancellation_timings(bad, 4) + + def test_source_audit_requires_one_complete_zero_record(self): + clean = "[info] [CookedSourceAudit] vfs_denied=0 registry_denied=0 imports_denied=0\n" + require_zero_source_audit(clean) + for bad in ("", clean + clean, clean.replace("imports_denied=0", "imports_denied=01"), + clean.replace("vfs_denied=0", "vfs_denied=2"), clean.rstrip() + "oops\n"): + with self.subTest(bad=bad), self.assertRaises(ValueError): + require_zero_source_audit(bad) + + def test_failure_retry_requires_the_original_scene_identity(self): + log = (f"[CookedRuntime] Published cooked scene generation 2 scene={SCENE}\n" + f"[CookedRuntime] Published cooked scene generation 3 scene={SCENE}\n") + validate_retry_publications(log) + with self.assertRaisesRegex(ValueError, "unexpected scene or generation"): + validate_retry_publications(log.replace(f"generation 3 scene={SCENE}", f"generation 3 scene={ALTERNATE_SCENE}")) + with self.assertRaisesRegex(ValueError, "unexpected scene or generation"): + validate_retry_publications(log + f"[CookedRuntime] Published cooked scene generation 3 scene={SCENE}\n") + + def test_executable_hash_covers_file_contents(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "binary" + path.write_bytes(b"first") + original = sha256_file(path) + path.write_bytes(b"second") + self.assertNotEqual(original, sha256_file(path)) + + def test_stress_rejects_stale_scene_and_wrong_cancel_generation(self): + log = (f"[CookedRuntime] Published cooked scene generation 2 scene={SCENE}\n" + "[CookedStress] cancelled=1 generation=2 rendered_after_cleanup=1\n" + f"[CookedRuntime] Published cooked scene generation 3 scene={ALTERNATE_SCENE}\n" + "[CookedStress] cancelled=2 generation=3 rendered_after_cleanup=1\n" + f"[CookedRuntime] Published cooked scene generation 4 scene={SCENE}\n") + validate_stress_sequence(log, 4) + with self.assertRaisesRegex(ValueError, "unexpected scene"): + validate_stress_sequence(log.replace(f"generation 3 scene={ALTERNATE_SCENE}", f"generation 3 scene={SCENE}"), 4) + with self.assertRaisesRegex(ValueError, "expected scene generation"): + validate_stress_sequence(log.replace("cancelled=2 generation=3", "cancelled=2 generation=2"), 4) + + def test_stress_rejects_resize_after_baseline_but_not_startup(self): + header = "[ZUI] UIScale=2.00 Screen=1280x720 Window=1280x720 Framebuffer=2560x1440\n" + dimensions = require_stable_stress_window("Window size updated\n" + header + "[CookedTiming] generation=2 admission_ns=1\n") + self.assertEqual(dimensions["framebuffer"], [2560, 1440]) + with self.assertRaisesRegex(ValueError, "viewport changed"): + require_stable_stress_window(header + "[CookedTiming] generation=2 admission_ns=1\nSwapchain recreated: 10x10\n") + self.assertTrue(observed_timing_viewport(header + "[CookedTiming] generation=2 admission_ns=1\nWindow size updated\n")["resize_after_baseline"]) + self.assertIsNone(observed_timing_viewport("[CookedTiming] generation=2 admission_ns=1\n")) + + def test_stress_rejects_engine_errors_and_environment_fallback(self): + require_clean_stress_log("[info] normal operation\n") + with self.assertRaisesRegex(ValueError, "engine error"): + require_clean_stress_log("[ERROR] resource failed\n") + with self.assertRaisesRegex(ValueError, "environment fallback"): + require_clean_stress_log("[warning] [SkyEnvironment] revision is using the fallback\n") + + def test_console_validation_errors_cannot_be_hidden_by_clean_engine_log(self): + require_clean_vulkan_console("normal process output\n") + with self.assertRaisesRegex(ValueError, "Vulkan console validation failure"): + require_clean_vulkan_console("Vulkan validation error: missing driver library\n") + with self.assertRaisesRegex(ValueError, "Vulkan console validation failure"): + require_clean_vulkan_console("VUID-vkQueueSubmit-invalid\n") + + def test_gpu_completion_requires_matching_generation_and_timeline(self): + log = ("[CookedLifecycle] generation=2 cpu_live=1 cpu_peak=1 staging_live=1 staging_peak=1 " + "metadata_live=1 metadata_blocks=1 gpu_live=1 gpu_peak=1 retirement_queued=0 retirement_draining=0\n" + "[CookedCache] generation=2 closures=0 leases=0 retiring=0 asset_versions=0\n" + "[CookedTiming] generation=2 admission_ns=10 first_submission_ns=20 elapsed_ns=10\n") + with self.assertRaisesRegex(RuntimeError, "missing render-completion"): + read_lifecycle_observations(log, [2], True) + valid = log + "[CookedGpuCompletion] generation=2 observed_ns=25 timeline_value=4\n" + self.assertEqual(read_lifecycle_observations(valid, [2], True)[0]["admission_to_completion_observed_ns"], 15) + with self.assertRaisesRegex(RuntimeError, "invalid render-completion"): + read_lifecycle_observations(valid.replace("timeline_value=4", "timeline_value=0"), [2], True) + with self.assertRaisesRegex(RuntimeError, "missing render-completion"): + read_lifecycle_observations(valid.replace("generation=2 observed_ns", "generation=3 observed_ns"), [2], True) + + def test_editor_lifecycle_does_not_claim_standalone_cache_observations(self): + log = ("[CookedLifecycle] generation=2 cpu_live=1 cpu_peak=1 staging_live=1 staging_peak=1 " + "metadata_live=1 metadata_blocks=1 gpu_live=1 gpu_peak=1 retirement_queued=0 retirement_draining=0\n" + "[CookedTiming] generation=2 admission_ns=10 first_submission_ns=20 elapsed_ns=10\n") + with self.assertRaisesRegex(RuntimeError, "cache observations"): + read_lifecycle_observations(log, [2]) + result = read_lifecycle_observations(log, [2], require_cache=False) + self.assertNotIn("cache_closures", result[0]) + + def test_reassembles_rotated_segments_in_chronological_order(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, content in (("engine_dump.2.log", "old\n"), ("engine_dump.1.log", "middle\n"), ("engine_dump.log", "new\n")): + (root / name).write_text(content, encoding="utf-8") + self.assertEqual(read_process_logs(root), "old\nmiddle\nnew\n") + + def test_rejects_missing_or_possibly_overwritten_segments(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "engine_dump.log").write_text("current", encoding="utf-8") + (root / "engine_dump.2.log").write_text("old", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "missing rotated segment"): + read_process_logs(root) + (root / "engine_dump.5.log").write_text("older", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "rotation limit"): + read_process_logs(root) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/scene/unit/test_cooked_timing_samples.py b/tests/scene/unit/test_cooked_timing_samples.py new file mode 100644 index 000000000..3ef5d91ad --- /dev/null +++ b/tests/scene/unit/test_cooked_timing_samples.py @@ -0,0 +1,67 @@ +import copy +from pathlib import Path +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).parents[1])) +from analyze_cooked_timing_samples import analyze + + +def sample(generation, elapsed): + return {"generation": generation, "admission_ns": 100, "first_submission_ns": 100 + elapsed, + "admission_to_submission_ns": elapsed, "render_completion_observed_ns": 200 + elapsed, + "render_timeline_value": generation, "admission_to_completion_observed_ns": elapsed + 100} + + +class CookedTimingSamplesTest(unittest.TestCase): + def setUp(self): + common = {"manifest_sha256": "a" * 64, "binary_sha256": {"runtime_sha256": "b" * 64, "cooker_sha256": "c" * 64}, + "host_system": "fixture", "host_machine": "test", "filesystem_cache_conditions": "uncontrolled", "source_audit_enabled": True, "gpu_completion_observed": True} + viewport = {"scene_viewport": [1280, 720], "window": [1280, 720], "framebuffer": [2560, 1440], "resize_after_baseline": False} + self.fresh = {**common, "fresh_process_samples": [sample(2, 1000), sample(2, 2000)], "validation_states": [True, True], "timing_viewports": [viewport, viewport]} + self.replacements = {**common, "samples": [sample(2, 1000), sample(3, 3000), sample(4, 4000)], + "reloads": 2, "validation_states": [True], "timing_viewports": [viewport]} + + def test_keeps_fresh_initial_and_replacement_samples_separate(self): + result = analyze(self.fresh, self.replacements, 2) + self.assertEqual(result["fresh_process_loads"]["count"], 2) + self.assertEqual(result["same_process_initial_load"]["count"], 1) + self.assertEqual(result["same_process_replacements"]["max_ms"], 0.004) + self.assertEqual(result["same_process_first_replacement"]["max_ms"], 0.003) + self.assertEqual(result["same_process_later_replacements"]["max_ms"], 0.004) + + def test_rejects_different_package_validation_or_invalid_timing(self): + other = copy.deepcopy(self.replacements) + other["manifest_sha256"] = "d" * 64 + with self.assertRaisesRegex(ValueError, "manifest"): + analyze(self.fresh, other, 2) + other = copy.deepcopy(self.replacements) + other["validation_states"] = [False] + with self.assertRaisesRegex(ValueError, "validation"): + analyze(self.fresh, other, 2) + other = copy.deepcopy(self.replacements) + other["samples"][1]["admission_to_submission_ns"] = 0 + with self.assertRaisesRegex(ValueError, "invalid admission"): + analyze(self.fresh, other, 2) + other = copy.deepcopy(self.replacements) + other["timing_viewports"][0]["resize_after_baseline"] = True + with self.assertRaisesRegex(ValueError, "viewport"): + analyze(self.fresh, other, 2) + + def test_rejects_mismatched_or_partial_recorded_device_identity(self): + self.fresh["shader_cache"] = {"sha256": "a" * 64, "files": 44} + with self.assertRaisesRegex(ValueError, "shader_cache"): + analyze(self.fresh, self.replacements, 2) + self.replacements["shader_cache"] = {"sha256": "b" * 64, "files": 44} + with self.assertRaisesRegex(ValueError, "shader_cache"): + analyze(self.fresh, self.replacements, 2) + self.replacements["shader_cache"] = copy.deepcopy(self.fresh["shader_cache"]) + self.assertEqual(analyze(self.fresh, self.replacements, 2)["shader_cache"], self.fresh["shader_cache"]) + self.fresh["shader_cache"]["files"] = 0 + self.replacements["shader_cache"]["files"] = 0 + with self.assertRaisesRegex(ValueError, "invalid shader_cache"): + analyze(self.fresh, self.replacements, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/scene/unit/test_sampleproject_cooked_timing.py b/tests/scene/unit/test_sampleproject_cooked_timing.py new file mode 100644 index 000000000..ed3b67517 --- /dev/null +++ b/tests/scene/unit/test_sampleproject_cooked_timing.py @@ -0,0 +1,99 @@ +import sys +from pathlib import Path +import unittest + +sys.path.insert(0, str(Path(__file__).parents[1])) +from sampleproject_cooked_timing import ALTERNATE, CAMERA, SCENE, device_identity, memory_summary, require_visible_samples, require_visible_trace, scene_pair, shader_identity + + +class SampleProjectTimingTest(unittest.TestCase): + def test_visible_samples_require_ordered_visible_state_through_completion(self): + samples = [{"admission_ns": 20, "render_completion_observed_ns": 40}] + require_visible_samples("[CookedWindowVisibility] timestamp_ns=10 visible=1\n", samples) + self.assertTrue(samples[0]["visible_window_verified"]) + with self.assertRaisesRegex(ValueError, "throughout"): + require_visible_samples("[CookedWindowVisibility] timestamp_ns=10 visible=1\n[CookedWindowVisibility] timestamp_ns=30 visible=0\n", samples) + with self.assertRaisesRegex(ValueError, "malformed"): + require_visible_samples("[CookedWindowVisibility] timestamp_ns=10 visible=3\n", samples) + with self.assertRaisesRegex(ValueError, "ordered"): + require_visible_samples("[CookedWindowVisibility] timestamp_ns=10 visible=1\n[CookedWindowVisibility] timestamp_ns=10 visible=0\n", samples) + + def test_visible_trace_rejects_occluded_or_unknown_windows(self): + visible = {"window_occlusion": {"visible_at_start": 1, "changes": []}} + windows = {name: visible for name in ("initial_load", "steady", "replacement", "post_steady")} + require_visible_trace({"windows": windows}) + with self.assertRaisesRegex(ValueError, "replacement"): + require_visible_trace({"windows": {**windows, "replacement": {"window_occlusion": {"visible_at_start": 1, "changes": [{"visible": 0}]}}}}) + with self.assertRaisesRegex(ValueError, "steady"): + require_visible_trace({"windows": {**windows, "steady": {"window_occlusion": None}}}) + with self.assertRaisesRegex(ValueError, "all four"): + require_visible_trace({"windows": {}}) + + def test_memory_summary_preserves_peak_and_sampled_scopes(self): + sample = {"cpu_peak": 10, "staging_peak": 20, "gpu_peak": 30, "metadata_live": 4, + "metadata_blocks": 1, "retirement_queued": 2, "retirement_draining": 0} + result = memory_summary([sample, {**sample, "cpu_peak": 15}]) + self.assertEqual(result["cpu_pool_peak_bytes"], 15) + self.assertEqual(result["retirement_max_queued_at_observation"], 2) + with self.assertRaisesRegex(ValueError, "requires"): + memory_summary([]) + with self.assertRaisesRegex(ValueError, "invalid"): + memory_summary([{**sample, "gpu_peak": -1}]) + + def test_device_identity_requires_unique_device_and_driver(self): + import tempfile + with tempfile.TemporaryDirectory() as directory: + driver = Path(directory) / "driver.bin" + driver.write_bytes(b"driver") + log = f'Using "GPU" with driver: "{driver}"\nCreated VkDevice to run on GPU GPU with the following 2 extensions\n' + self.assertEqual(device_identity(log)["gpu"], "GPU") + self.assertIsNone(device_identity(log + log)) + self.assertIsNone(device_identity("no device information")) + + def test_memory_summary_requires_complete_budget_observations(self): + sample = {"cpu_peak": 10, "staging_peak": 20, "gpu_peak": 30, "metadata_live": 4, + "metadata_blocks": 1, "retirement_queued": 2, "retirement_draining": 0, + "gpu_admitted_peak": 100, "gpu_geometry_peak": 20, "gpu_texture_peak": 30, + "gpu_target_peak": 10, "gpu_uniform_peak": 5, "gpu_staging_peak": 15, + "gpu_readback_peak": 0, "retirement_high_water": 4, "retirement_capacity": 16} + result = memory_summary([sample, {**sample, "gpu_admitted_peak": 120, "retirement_high_water": 7}]) + self.assertEqual(result["gpu_admitted_peak_bytes"], 120) + self.assertEqual(result["gpu_texture_peak_bytes"], 30) + self.assertEqual(result["retirement_high_water"], 7) + self.assertEqual(result["retirement_capacity"], 16) + with self.assertRaisesRegex(ValueError, "partial GPU"): + memory_summary([sample, {k: v for k, v in sample.items() if k != "gpu_admitted_peak"}]) + with self.assertRaisesRegex(ValueError, "capacity changed"): + memory_summary([sample, {**sample, "retirement_capacity": 32}]) + + def test_shader_identity_tracks_names_and_contents(self): + import tempfile + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with self.assertRaisesRegex(ValueError, "missing"): + shader_identity(root) + path = root / "fixture.spv" + path.write_bytes(b"first") + original = shader_identity(root) + path.write_bytes(b"second") + changed = shader_identity(root) + self.assertNotEqual(changed, original) + path.rename(root / "renamed.spv") + self.assertNotEqual(shader_identity(root), changed) + + def test_pair_preserves_source_and_adds_unique_scene_identity_and_camera(self): + original = ('"scene":\n "uuid": "' + SCENE + '"\n').encode() + initial, alternate = scene_pair(original) + self.assertTrue(initial.startswith(original)) + self.assertIn(CAMERA.encode(), initial) + self.assertIn(ALTERNATE.encode(), alternate) + self.assertNotIn(SCENE.encode(), alternate) + self.assertEqual(initial.count(CAMERA.encode()), 1) + with self.assertRaises(ValueError): + scene_pair(initial) + with self.assertRaises(ValueError): + scene_pair(original + original) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/scene/unit/test_sampleproject_runtime_reference.py b/tests/scene/unit/test_sampleproject_runtime_reference.py new file mode 100644 index 000000000..7886b6edd --- /dev/null +++ b/tests/scene/unit/test_sampleproject_runtime_reference.py @@ -0,0 +1,124 @@ +from pathlib import Path +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).parents[1])) +from sampleproject_cooked_runtime import add_sampleproject_hierarchy, compare_reviewed_reference, count_dark_pixels, force_opaque_materials, parse_presence, remove_sampleproject_directional_light, review_result, validate_control_baseline, validation_layer_status + +SCENE_FIXTURE = b'''"scene": + "entities": + - "uuid": "beb33379-561e-4413-af36-b7d9f5a6be26" + "components": + "TransformComponent": + "fields": + "position": [0, 0, 0] + - "uuid": "c879cf96-693a-46f5-a772-ab927a72215b" + "components": + "LightComponent": + "fields": + "intensity": 3 + - "uuid": "e97f8f91-1fe7-415a-89ec-9e67e04ad960" + "components": + "TransformComponent": + "fields": + "position": [0.40000004, 1.2, 0] +''' + + +class ReviewedReferenceTest(unittest.TestCase): + def test_control_baseline_requires_matching_device_build_and_pixels(self): + import hashlib + baseline = (1, 1, b"abc") + binaries, shaders, device = {"runtime": "a"}, {"sha256": "b"}, {"gpu": "c"} + evidence = {"binary_sha256": binaries, "shader_cache": shaders, "device": device, + "source_pixel_sha256": hashlib.sha256(baseline[2]).hexdigest()} + validate_control_baseline(evidence, baseline, binaries, shaders, device) + with self.assertRaisesRegex(ValueError, "identity"): + validate_control_baseline(evidence, baseline, binaries, shaders, {"gpu": "different"}) + with self.assertRaisesRegex(ValueError, "pixels"): + validate_control_baseline(evidence, (1, 1, b"def"), binaries, shaders, device) + with self.assertRaisesRegex(ValueError, "identity"): + validate_control_baseline({**evidence, "device": None}, baseline, binaries, shaders, None) + + def test_directional_light_control_requires_one_known_intensity(self): + scene = SCENE_FIXTURE + result = remove_sampleproject_directional_light(scene) + self.assertIn(b'"intensity": 0\n', result) + self.assertNotIn(b'"intensity": 3\n', result) + with self.assertRaisesRegex(ValueError, "directional light"): + remove_sampleproject_directional_light(result) + + def test_opaque_control_only_changes_blend_materials(self): + import json + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + blend = root / "blend.zematerial" + opaque = root / "opaque.zematerial" + blend.write_text('{"alpha_mode": "BLEND", "value": 1}') + opaque.write_text('{"alpha_mode": "OPAQUE", "value": 2}') + unchanged = opaque.read_bytes() + changed = force_opaque_materials(root) + self.assertEqual(len(changed), 1) + self.assertEqual(json.loads(blend.read_text()), {"alpha_mode": "OPAQUE", "value": 1}) + self.assertEqual(opaque.read_bytes(), unchanged) + with self.assertRaisesRegex(ValueError, "no blend materials"): + force_opaque_materials(root) + + def test_hierarchy_fixture_moves_translation_and_rejects_reapplication(self): + scene = SCENE_FIXTURE + result = add_sampleproject_hierarchy(scene) + self.assertEqual(result.count(b'"parent_uuid": "beb33379-561e-4413-af36-b7d9f5a6be26"'), 1) + self.assertEqual(result.count(b'"position": [0.40000004, 1.2, 0]'), 1) + with self.assertRaisesRegex(ValueError, "hierarchy fixture"): + add_sampleproject_hierarchy(result) + + def test_named_region_detects_small_change_hidden_by_viewport_tolerance(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "reference.ppm" + pixels = bytes(20 * 20 * 3) + path.write_bytes(b"P6\n20 20\n255\n" + pixels) + changed = bytearray(pixels) + changed[0] = 255 + result = compare_reviewed_reference(path, (20, 20, bytes(changed)), [("important", (0, 0, 1, 1))]) + self.assertTrue(result["viewport"]["passed"]) + self.assertFalse(result["regions"]["important"]["passed"]) + + def test_rejects_reference_with_different_dimensions(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "reference.ppm" + path.write_bytes(b"P6\n1 1\n255\n" + bytes(3)) + with self.assertRaisesRegex(ValueError, "dimensions differ"): + compare_reviewed_reference(path, (2, 1, bytes(6)), []) + result = review_result(path, (2, 1, bytes(6)), [], "review-1") + self.assertIn("dimensions differ", result["error"]) + self.assertEqual(result["review_identifier"], "review-1") + + def test_presence_counts_only_dark_pixels_in_declared_region(self): + capture = (2, 2, bytes((0, 0, 0, 99, 99, 99, 100, 0, 0, 255, 255, 255))) + self.assertEqual(count_dark_pixels(capture, (0, 0, 2, 1)), 2) + self.assertEqual(count_dark_pixels(capture, (0, 1, 2, 1)), 0) + self.assertEqual(parse_presence("character:1000"), ("character", 1000)) + with self.assertRaises(ValueError): + count_dark_pixels(capture, (1, 1, 2, 1)) + + def test_validation_status_requires_one_explicit_record(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "engine.log" + path.write_text("[VulkanValidation] khronos_enabled=1\n", encoding="utf-8") + self.assertTrue(validation_layer_status(path)) + path.write_text("[VulkanValidation] khronos_enabled=0\n", encoding="utf-8") + self.assertFalse(validation_layer_status(path)) + path.write_text("unrelated log\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "missing or repeated"): + validation_layer_status(path) + path.write_text("[VulkanValidation] khronos_enabled=1x\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "malformed"): + validation_layer_status(path) + path.write_text("[VulkanValidation] khronos_enabled=1\n[VulkanValidation] khronos_enabled=0\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "missing or repeated"): + validation_layer_status(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/Scripts/tests/test_scene_capture.py b/tests/scene/unit/test_scene_capture.py similarity index 85% rename from Scripts/tests/test_scene_capture.py rename to tests/scene/unit/test_scene_capture.py index bb36ee46a..1d622c028 100644 --- a/Scripts/tests/test_scene_capture.py +++ b/tests/scene/unit/test_scene_capture.py @@ -1,4 +1,5 @@ import importlib.util +import gzip from pathlib import Path import tempfile import unittest @@ -23,6 +24,12 @@ def test_crlf_separator(self): self.path.write_bytes(b"P6\r\n1 1\r\n255\r\n\n\r\t") self.assertEqual(capture.read_ppm(self.path), (1, 1, b"\n\r\t")) + def test_lossless_gzip_reference_uses_the_same_bounded_parser(self): + compressed = self.path.with_suffix(".ppm.gz") + with gzip.open(compressed, "wb") as output: + output.write(b"P6\n1 1\n255\n\x01\x02\x03") + self.assertEqual(capture.read_ppm(compressed), (1, 1, b"\x01\x02\x03")) + def test_malformed_or_wrong_space_capture_is_rejected(self): for data in (b"", b"P3\n1 1\n255\n123", b"P6\n0 1\n255\n", b"P6\n1 1\n65535\n123456", b"P6\n1 1\n255\n12", b"P6\n1 1\n255\n1234", b"P6\n# missing newline"): with self.subTest(data=data): @@ -80,6 +87,11 @@ def test_capture_log_requires_complete_nonempty_clean_shutdown(self): self.path.write_text("[info] Engine initialized\n[info] Engine destroyed\n", encoding="utf-8") capture.validate_capture_log(self.path) + def test_reconstructed_log_rejects_errors_in_earlier_segments(self): + capture.validate_capture_log_text("[info] first segment\n[info] Engine destroyed\n") + with self.assertRaisesRegex(ValueError, "validation errors"): + capture.validate_capture_log_text("[error] first segment failed\n[info] Engine destroyed\n") + if __name__ == "__main__": unittest.main() diff --git a/Scripts/tests/test_scene_lifecycle.py b/tests/scene/unit/test_scene_lifecycle.py similarity index 96% rename from Scripts/tests/test_scene_lifecycle.py rename to tests/scene/unit/test_scene_lifecycle.py index 8afa206ed..ddcd8283e 100644 --- a/Scripts/tests/test_scene_lifecycle.py +++ b/tests/scene/unit/test_scene_lifecycle.py @@ -10,6 +10,15 @@ class LifecycleEvidenceTest(unittest.TestCase): + def test_capture_accepts_only_declared_occlusion_states(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "stages.csv" + path.write_text('kind,name,timestamp_ns,value\nvalue,SceneLoad.Main.WindowOcclusionVisible,1,1\n') + self.assertEqual(list(lifecycle.capture_records(path))[0][3], 1) + path.write_text('kind,name,timestamp_ns,value\nvalue,SceneLoad.Main.WindowOcclusionVisible,1,3\n') + with self.assertRaisesRegex(ValueError, "invalid window occlusion"): + list(lifecycle.capture_records(path)) + def setUp(self): self.directory = tempfile.TemporaryDirectory() self.addCleanup(self.directory.cleanup) diff --git a/tests/scene/unit/test_scene_sanitizer_results.py b/tests/scene/unit/test_scene_sanitizer_results.py new file mode 100644 index 000000000..0bcc65671 --- /dev/null +++ b/tests/scene/unit/test_scene_sanitizer_results.py @@ -0,0 +1,32 @@ +from pathlib import Path +import sys +import tempfile +import unittest +import xml.etree.ElementTree as ET + +sys.path.insert(0, str(Path(__file__).parents[1])) +from verify_scene_sanitizer_results import REQUIRED, verify + + +class SceneSanitizerResultsTest(unittest.TestCase): + def test_requires_every_suite_and_rejects_skipped_cases(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "results.xml" + root = ET.Element("testsuites") + for name in sorted(REQUIRED): + suite = ET.SubElement(root, "testsuite", name=name, tests="1", skipped="0", failures="0", errors="0", disabled="0") + ET.SubElement(suite, "testcase", name="case", status="run") + ET.ElementTree(root).write(path) + self.assertEqual(verify(path), len(REQUIRED)) + root[0][0].set("status", "notrun") + ET.ElementTree(root).write(path) + with self.assertRaisesRegex(ValueError, "did not run"): + verify(path) + root.remove(root[-1]) + ET.ElementTree(root).write(path) + with self.assertRaisesRegex(ValueError, "missing"): + verify(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/Scripts/validate_scene_lifecycle.py b/tests/scene/validate_scene_lifecycle.py similarity index 98% rename from Scripts/validate_scene_lifecycle.py rename to tests/scene/validate_scene_lifecycle.py index b8bdeb007..b1bf8ec4f 100644 --- a/Scripts/validate_scene_lifecycle.py +++ b/tests/scene/validate_scene_lifecycle.py @@ -182,11 +182,14 @@ def capture_records(path, window=None): continue if row["kind"] == "value": memory = row["name"].startswith("SceneLoad.Memory.") and row["name"].endswith("Bytes") + visibility = row["name"] == "SceneLoad.Main.WindowOcclusionVisible" + if visibility and value not in (0, 1, 2): + raise ValueError("invalid window occlusion observation") if memory and (not value.is_integer() or value > 2 ** 53 or Decimal(row["value"]) != int(value)): raise ValueError("inexact memory byte count") if row["name"].startswith("SceneLoad.Memory.") and not memory: raise ValueError("unknown memory units") - if not memory and not row["name"].endswith("Ms"): + if not memory and not visibility and not row["name"].endswith("Ms"): raise ValueError("unknown value units") yield row["kind"], row["name"], timestamp, value diff --git a/tests/scene/verify_scene_sanitizer_results.py b/tests/scene/verify_scene_sanitizer_results.py new file mode 100644 index 000000000..11db884fd --- /dev/null +++ b/tests/scene/verify_scene_sanitizer_results.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Reject empty, skipped or failed scene sanitizer test suites.""" +import argparse +from pathlib import Path +import sys +import xml.etree.ElementTree as ET + + +REQUIRED = frozenset({"CookedObjectSnapshotTest", "CookedPackageSnapshotTest", "SceneLoadCoordinatorTest", + "SceneRuntimeCameraTest", "SceneSubmissionProgressTest", "SceneInstanceTest", + "SceneResourceBundleTest", "SceneEnvironmentHandoffTest"}) + + +def verify(path): + root = ET.parse(path).getroot() + if root.tag != "testsuites": + raise ValueError("invalid Google Test report") + suites = root.findall("testsuite") + names = [suite.get("name") for suite in suites] + if len(names) != len(set(names)) or set(names) != REQUIRED: + raise ValueError("missing, repeated or unexpected scene test suite") + for suite in suites: + cases = suite.findall("testcase") + if not cases or int(suite.get("tests", "-1")) != len(cases): + raise ValueError("scene test suite has no complete test results") + if any(int(suite.get(key, "0")) != 0 for key in ("failures", "errors", "skipped", "disabled")): + raise ValueError("scene test suite contains failed, skipped or disabled tests") + if any(case.get("status") != "run" or case.find("skipped") is not None or case.find("failure") is not None for case in cases): + raise ValueError("scene test did not run successfully") + return sum(len(suite.findall("testcase")) for suite in suites) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("report", type=Path) + args = parser.parse_args() + print(f"Verified {verify(args.report)} executed scene sanitizer tests") + + +if __name__ == "__main__": + try: + main() + except (OSError, ET.ParseError, ValueError) as error: + print(f"Scene sanitizer result validation failed: {error}", file=sys.stderr) + sys.exit(1)