diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db2ac26..30a0ba0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,6 +77,7 @@ jobs: "git" "lld" "llvm" + "nasm" ) moonlight_dependencies=( ) @@ -92,6 +93,7 @@ jobs: "coreutils" "lld" "llvm" + "nasm" ) moonlight_dependencies=( ) @@ -115,6 +117,7 @@ jobs: mingw-w64-x86_64-lld mingw-w64-x86_64-llvm mingw-w64-x86_64-make + mingw-w64-x86_64-nasm - name: Setup python id: setup-python @@ -157,14 +160,17 @@ jobs: - name: Run tests id: test - working-directory: ${{ env.CMAKE_BUILD }}/tests - run: ./test_moonlight --gtest_color=yes --gtest_output=xml:test_results.xml + run: | + ctest --test-dir "${CMAKE_BUILD}" --output-on-failure \ + --output-junit "${PWD}/${CMAKE_BUILD}/tests/test_results.xml" - name: Generate gcov report id: test_report + # The Windows Clang host build does not enable gcov instrumentation. # any except canceled or skipped if: >- always() && + runner.os != 'Windows' && (steps.test.outcome == 'success' || steps.test.outcome == 'failure') env: MSYS2_PATH_TYPE: inherit @@ -188,7 +194,7 @@ jobs: # any except canceled or skipped if: >- always() && - (steps.test.outcome == 'success' || steps.test.outcome == 'failure') && + steps.test_report.outcome == 'success' && startsWith(github.repository, 'SunshineStream/') uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: diff --git a/AGENTS.md b/AGENTS.md index d5a4722..a4d0b35 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,29 +1,120 @@ -On Windows we use msys2 and mingw64 to compile. -You need to prefix commands with `C:\msys64\msys2_shell.cmd -defterm -here -no-start -mingw64 -c`. +# Agent instructions -Prefix build directories with `cmake-build-`. +## Global instructions -The xbox build uses Unix Makefiles. +Before working on a task, read `~/.agents/skills/AGENTS.md`, expanding `~` to the current user's home directory. +If that file is unavailable, report that the shared global instructions could not be loaded. -The host native tests will use MinGW Makefiles on Windows, but Unix Makefiles on other platforms. +## Repository boundaries -The project uses gtest as a test framework. +This repository is the original Xbox Moonlight client, using nxdk, SDL and FFmpeg. The adjacent Sunshine +checkout is a separate project; use it as a reference when relevant, and keep changes in this checkout +unless the task explicitly includes another repository. -Always add or update doxygen documentation. +Preserve unrelated working-tree changes. Treat `third-party/` as vendored dependencies. Apply Xbox +compatibility adaptations through the project's adapters and build-copy mechanisms rather than editing +generated SDK copies or dependency outputs. Keep generated files and local emulator support data under +their existing build or ignored directories. -The project requires that everything be documented in doxygen or the build will fail. +## Build -Primary doxygen comments should be done like so: +On Windows, compile with MSYS2 MinGW64. Prefix Windows build and test commands with +`C:\msys64\msys2_shell.cmd -defterm -here -no-start -mingw64 -c`. + +Prefix local build directories with `cmake-build-`. Host builds and native tests use MinGW Makefiles on +Windows and Unix Makefiles on other platforms. Xbox builds use Unix Makefiles on every platform. + +The normal top-level configure uses the host toolchain and delegates Xbox compilation to an nxdk child +build. Use `CMakePresets.json` and the README's build instructions; keep the Xbox toolchain separate from +host-native tests. The normal release ISO is `cmake-build-release/xbox/Moonlight.iso`, with the packaged +Xbox executable at `cmake-build-release/xbox/xbe/default.xbe`. + +Video experiment builds are opt-in through `MOONLIGHT_BUILD_VIDEO_EXPERIMENTS`. Keep their instructions +with the probes in `experiments/README.md`. + +## Tests and validation + +The project uses GoogleTest. Tests must include `` instead of including +GoogleTest directly. This supplies the shared test support and output capture; Sunshine's +`tests/tests_common.h` is not part of this repository. + +Group test includes under `// test includes`, then `// standard includes`, `// lib includes` and +`// local includes`, with blank lines between groups and no empty groups. Keep platform-specific headers +inside their preprocessor guards. Preserve and explain intentional include-order dependencies. + +Add or update meaningful regression tests for new or changed source behavior. Target full coverage of +changed testable source code. Keep platform-neutral logic accessible to native tests and document any +validation limits that require an emulator or console. + +Run tests with `ctest --test-dir --output-on-failure` so every registered test executable +is included. Native test executables, including `test_moonlight` and `test_pvideo_presenter`, are under the +build directory's `tests/` folder. A host-only configuration can disable Xbox builds with `BUILD_XBOX=OFF`. + +Build-system changes require appropriate configuration, build and lint checks rather than dedicated +build-system regression projects or fixtures. Documentation-only edits require a strict documentation +build and relevant formatting/link checks, rather than new source tests. + +For presentation changes, check GPU and software paths. Distinguish native test results, emulator +behavior and real-console validation; emulator output does not establish console performance or GPU +image correctness. Preserve software fallback when GPU presentation cannot be used. + +## Python tooling and formatting + +From the repository root, use `uv sync --locked` to refresh `.venv` from `pyproject.toml` and `uv.lock`. +Re-sync after switching branches or changing those files or `third-party/lizardbyte-common`. Update the +lockfile with `uv lock` when changing dependency declarations or the shared dependency revision. + +Follow `.clang-format` for C and C++. After native source changes, run +`uv run --locked lb-update-clang-format` from the repository root and inspect the diff for unrelated +formatting changes. Format changed C/C++ files in `experiments/` separately: the shared formatter does +not traverse that directory. Centrally managed style files should be updated through their owning +configuration repository rather than changed locally. + +## Code documentation + +Always add or update Doxygen documentation for source changes. Document new or changed functions, +classes, structures, members, test fixtures and tests. The strict documentation build fails on missing +required documentation. + +Use primary Doxygen comments in this form: ```cpp - /** - * @brief Describe the function, structure, etc. - * - * @param my_param Describe the parameter. - * @return Describe the return. - */ +/** + * @brief Describe the function, structure, etc. + * + * @param my_param Describe the parameter. + * @return Describe the return. + */ ``` -Inline doxygen comments should use `///< ...` instead of `/**< ... */`. +Use `///< ...` for inline Doxygen comments, rather than `/**< ... */`. + +Keep implementation constraints and reasons for compatibility workarounds beside the code that depends +on them. Keep user guidance in the README and metric definitions in the corresponding API documentation. +Published documentation should describe supported behavior and operating instructions, rather than +debugging chronologies, session-specific frame counts or research transcripts. + +Documentation is built through Dockle and Doxygen using `dockle.toml`. Validate documentation edits with +the strict `api` build; a CMake build configured with `BUILD_DOCS=ON` provides the `docs` target. + +## Markdown style + +Align the pipes in Markdown tables in the source. Pad each column to a consistent width across the +header, separator and body rows, preserving any column-alignment markers. Apply this to all first-party +Markdown files, including READMEs and this file. + +Use relative repository links for local documentation. Do not add decorative separator comments or +dashed banners to group tests; use descriptive names and code structure. + +## Issues, pull requests and reviews + +Use applicable repository or inherited organization templates when creating issues or pull requests, +following the shared global instructions. For an issue-only request, investigate enough to verify and +describe the observed behavior, expected behavior and reproduction steps. Investigate root causes and +implement fixes fully when those are requested. -Always follow the style guidelines defined in .clang-format for c/c++ code. +Review the current pull request head and verify mergeability against its target branch. Report concrete +introduced problems with their trigger, impact, location and a practical fix. Keep workflow status and +validation limitations separate from code findings. Pending maintainer approval or checks that have not +run are verification limits, not defects for the contributor to fix. Review actual failed checks for +change-related problems. An approval does not need an additional comment solely to announce it. diff --git a/CMakeLists.txt b/CMakeLists.txt index 5fb5772..0c23d18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,8 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON CACHE BOOL "Export compile commands for IDE set(MOONLIGHT_BUILD_KIND "HOST" CACHE STRING "Internal Moonlight build mode") set_property(CACHE MOONLIGHT_BUILD_KIND PROPERTY STRINGS HOST XBOX) +option(MOONLIGHT_BUILD_VIDEO_EXPERIMENTS "Build standalone GPU presentation and MPEG-2 sampling experiments" OFF) +option(MOONLIGHT_ENABLE_GPU_PRESENTATION "Enable experimental NV2A overlay presentation before software fallback" ON) option(MOONLIGHT_INPUT_DIAGNOSTICS "Display digital controller state while diagnosing stream input" OFF) if(MOONLIGHT_BUILD_KIND STREQUAL "XBOX") diff --git a/README.md b/README.md index b25630d..0d5ab36 100644 --- a/README.md +++ b/README.md @@ -22,14 +22,26 @@ ![Apps Screen](./docs/images/screenshots/03-apps.png) -## Streaming controls +## Streaming controls and statistics To quit a stream, press **Back/Select + Black + White (LB + RB)** together on any connected controller. -The client displays **Stopping stream...** before returning to the menu. If the end-of-stream statistics -summary is enabled, release the buttons and press a button to dismiss it. +The client displays **Stopping stream...**, then a stream summary. Release the buttons, then press a +button to return to the menu. -The opt-in CMake option `MOONLIGHT_INPUT_DIAGNOSTICS` displays controller state and event counters -for troubleshooting; it is disabled by default. +Enable **Settings → Display → Performance Stats** for a live panel during playback and additional +performance details in the summary. The panel works with GPU and software presentation, refreshes +once per second, and shows `N/A` for unavailable measurements. It includes measured incoming, +decoded and rendered FPS, video bitrate, timing, frame drops, buffering and the presentation path. +The stream header shows the requested FPS; the three FPS counters show measured rates. +Network latency is round-trip time. Present measures CPU frame preparation and submission, +including any scanout wait. + +Video decoding runs on the CPU. Optional experimental GPU presentation accelerates color conversion +and scaling, with software fallback for unsupported frames or presentation failures. GPU +presentation is enabled by default; the build options below control its use. + +Every stream summary reports GPU and software frame counts. **SOFTWARE FALLBACK USED** includes +the reason software presentation was selected, even when Performance Stats is disabled. ## 🛠️ Build @@ -61,6 +73,7 @@ nxdk_dependencies=( "mingw-w64-x86_64-lld" "mingw-w64-x86_64-llvm" "mingw-w64-x86_64-make" + "mingw-w64-x86_64-nasm" ) moonlight_dependencies=( "mingw-w64-x86_64-doxygen" @@ -83,6 +96,7 @@ nxdk_dependencies=( "git" "lld" "llvm" + "nasm" ) moonlight_dependencies=( "doxygen" @@ -101,6 +115,7 @@ nxdk_dependencies=( "coreutils" "lld" "llvm" + "nasm" ) moonlight_dependencies=( "doxygen" @@ -121,6 +136,17 @@ For host and Xbox builds, CMake checks whether the compiler supports Clang's `un cmake -S . -B cmake-build-release -DBUILD_DOCS=OFF -DCMAKE_BUILD_TYPE=Release ``` +Video build settings: + +| CMake option | Default | Purpose | +| ----------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `MOONLIGHT_ENABLE_GPU_PRESENTATION` | `ON` | Enable experimental NV2A color conversion and scaling with software fallback. | +| `MOONLIGHT_BUILD_VIDEO_EXPERIMENTS` | `OFF` | Build the standalone probes described in [Video experiments](experiments/README.md). | +| `MOONLIGHT_INPUT_DIAGNOSTICS` | `OFF` | Show controller state and event counters for troubleshooting. Requires GPU presentation to be disabled so the strip is visible. | + +Pass options when configuring, for example `-DMOONLIGHT_ENABLE_GPU_PRESENTATION=OFF` for software-only +presentation. Existing build directories retain their cached settings until explicitly reconfigured. + ### Build ```bash diff --git a/cmake/moonlight-dependencies.cmake b/cmake/moonlight-dependencies.cmake index 7213926..734e221 100644 --- a/cmake/moonlight-dependencies.cmake +++ b/cmake/moonlight-dependencies.cmake @@ -1,6 +1,6 @@ include_guard(GLOBAL) -# Disable nxdk's incomplete MSVC aligned-allocation family and unimplemented math entry points. +# Disable nxdk's unimplemented math entry points; aligned allocation uses private adapters. function(_moonlight_patch_ffmpeg_config_header ffmpeg_config_header) if(NOT EXISTS "${ffmpeg_config_header}") return() @@ -10,7 +10,6 @@ function(_moonlight_patch_ffmpeg_config_header ffmpeg_config_header) foreach(ffmpeg_disabled_probe IN ITEMS - HAVE_ALIGNED_MALLOC HAVE_EXP2 HAVE_EXP2F HAVE_LLRINT @@ -29,6 +28,31 @@ function(_moonlight_patch_ffmpeg_config_header ffmpeg_config_header) file(WRITE "${ffmpeg_config_header}" "${ffmpeg_config_text}") endfunction() +# Adapt the upstream thread wrapper only in the build directory. The source +# include path follows the build path, so FFmpeg finds this private header first. +# nxdk condition variables own kernel events that require explicit destruction; +# desktop Windows thread-naming APIs are unavailable on the Xbox runtime. +function(_moonlight_prepare_ffmpeg_thread_header ffmpeg_source_dir ffmpeg_build_dir) + set(thread_header "${ffmpeg_source_dir}/compat/w32pthreads.h") + file(READ "${thread_header}" thread_text) + string(REPLACE "\r\n" "\n" thread_text "${thread_text}") + set(destroy_original "static inline int pthread_cond_destroy(pthread_cond_t *cond)\n{\n") + string(FIND "${thread_text}" "${destroy_original}" destroy_position) + string(FIND "${thread_text}" "#if !HAVE_UWP\n" naming_position) + if(destroy_position EQUAL -1 OR naming_position EQUAL -1) + message(FATAL_ERROR "FFmpeg's Windows thread wrapper changed; review the nxdk adaptations") + endif() + string(REPLACE "${destroy_original}" + "${destroy_original} UninitializeConditionVariable(cond);\n" + thread_text "${thread_text}") + string(REPLACE "/* native condition variables do not destroy */" + "/* nxdk condition variables own kernel events. */" thread_text "${thread_text}") + string(REPLACE "#if !HAVE_UWP\n" "#if !HAVE_UWP && !defined(NXDK)\n" + thread_text "${thread_text}") + file(MAKE_DIRECTORY "${ffmpeg_build_dir}/compat") + file(WRITE "${ffmpeg_build_dir}/compat/w32pthreads.h" "${thread_text}") +endfunction() + # Validate FFmpeg source and support files before attempting a configure. function(_moonlight_validate_xbox_ffmpeg_inputs ffmpeg_source_dir) if(NOT EXISTS "${ffmpeg_source_dir}/configure") @@ -68,6 +92,7 @@ function(_moonlight_compute_ffmpeg_signature out_var nxdk_dir ffmpeg_source_dir) file(READ "${nxdk_dir}/.moonlight-source.signature" nxdk_revision) file(SHA256 "${ffmpeg_cc_wrapper}" ffmpeg_cc_wrapper_hash) file(SHA256 "${ffmpeg_cxx_wrapper}" ffmpeg_cxx_wrapper_hash) + file(SHA256 "${CMAKE_SOURCE_DIR}/scripts/ffmpeg-nxdk-nasm.sh" ffmpeg_nasm_wrapper_hash) file(GLOB_RECURSE compat_inputs CONFIGURE_DEPENDS "${CMAKE_SOURCE_DIR}/src/_nxdk_compat/*.h" "${CMAKE_SOURCE_DIR}/src/_nxdk_compat/*.c") @@ -88,6 +113,7 @@ function(_moonlight_compute_ffmpeg_signature out_var nxdk_dir ffmpeg_source_dir) "FFMPEG_ARCH=x86" "FFMPEG_CC_WRAPPER_SHA256=${ffmpeg_cc_wrapper_hash}" "FFMPEG_CXX_WRAPPER_SHA256=${ffmpeg_cxx_wrapper_hash}" + "FFMPEG_NASM_WRAPPER_SHA256=${ffmpeg_nasm_wrapper_hash}" "FFMPEG_DEPENDENCY_MANIFEST_SHA256=${ffmpeg_dependency_manifest_hash}") list(JOIN signature_inputs "\n" signature_text) string(SHA256 signature "${signature_text}") @@ -157,15 +183,20 @@ function(_moonlight_get_ffmpeg_configure_args out_var) set(ffmpeg_install_shell_path "${ARGV2}") set(ffmpeg_cc_shell_path "sh ${ARGV3}") set(ffmpeg_cxx_shell_path "sh ${ARGV4}") + get_filename_component(ffmpeg_scripts_shell_path "${ARGV3}" DIRECTORY) + # Pentium III supports the NASM MMX/SSE kernels, but not SSE2 or AMD 3DNow. + # Win32 stack realignment leaves too few registers for FFmpeg's inline CABAC + # assembly; standalone NASM routines remain enabled. set(ffmpeg_configure_args sh "${ffmpeg_source_shell_path}/configure" "--prefix=${ffmpeg_install_shell_path}" "--extra-ldflags=${MOONLIGHT_NXDK_COMPAT_LIBRARY}" + --extra-cflags=-mstackrealign --enable-cross-compile --arch=x86 - --cpu=i686 + --cpu=pentium3 --target-os=none "--cc=${ffmpeg_cc_shell_path}" "--cxx=${ffmpeg_cxx_shell_path}" @@ -175,9 +206,14 @@ function(_moonlight_get_ffmpeg_configure_args out_var) --enable-static --disable-shared --disable-autodetect - --disable-asm + --enable-asm --disable-inline-asm - --disable-x86asm + --enable-x86asm + # FFmpeg probes executable names individually; this must not be a shell command. + "--x86asmexe=${ffmpeg_scripts_shell_path}/ffmpeg-nxdk-nasm.sh" + --disable-sse2 + --disable-amd3dnow + --disable-amd3dnowext --disable-debug --disable-doc --disable-programs @@ -194,7 +230,7 @@ function(_moonlight_get_ffmpeg_configure_args out_var) --disable-symver --disable-runtime-cpudetect --disable-pthreads - --disable-w32threads + --enable-w32threads --disable-os2threads --disable-hwaccels --enable-avcodec @@ -240,6 +276,7 @@ function(_moonlight_rebuild_xbox_ffmpeg nxdk_dir ffmpeg_source_dir ffmpeg_build_ file(REMOVE_RECURSE "${ffmpeg_build_dir}" "${ffmpeg_install_dir}") file(MAKE_DIRECTORY "${ffmpeg_build_dir}") + _moonlight_prepare_ffmpeg_thread_header("${ffmpeg_source_dir}" "${ffmpeg_build_dir}") _moonlight_get_ffmpeg_shell_paths( ffmpeg_shell_paths @@ -308,7 +345,8 @@ function(moonlight_prepare_xbox_ffmpeg nxdk_dir) _moonlight_validate_xbox_ffmpeg_inputs( "${ffmpeg_source_dir}" "${ffmpeg_cc_wrapper}" - "${ffmpeg_cxx_wrapper}") + "${ffmpeg_cxx_wrapper}" + "${CMAKE_SOURCE_DIR}/scripts/ffmpeg-nxdk-nasm.sh") set(ffmpeg_state_dir "${CMAKE_BINARY_DIR}/third-party/ffmpeg") set(ffmpeg_build_dir "${ffmpeg_state_dir}/build") diff --git a/cmake/nxdk-compat.cmake b/cmake/nxdk-compat.cmake index 5cc42fa..1cd4e92 100644 --- a/cmake/nxdk-compat.cmake +++ b/cmake/nxdk-compat.cmake @@ -75,10 +75,13 @@ function(moonlight_prepare_nxdk_compat) moonlight_run_nxdk_command("Compatibility probe object" "${NXDK_DIR}" "${port}" "${NXDK_DIR}/bin/nxdk-cc" "-I${compat}/include" -c "${compat}/runtime.c" -o runtime.obj) + moonlight_run_nxdk_command("FFmpeg allocation probe object" "${NXDK_DIR}" "${port}" + "${NXDK_DIR}/bin/nxdk-cc" + -c "${compat}/ffmpeg_memory.c" -o ffmpeg_memory.obj) moonlight_run_nxdk_command("Compatibility probe archive" "${NXDK_DIR}" "${port}" - llvm-ar rcs compat.lib runtime.obj) + llvm-ar rcs compat.lib runtime.obj ffmpeg_memory.obj) set(MOONLIGHT_NXDK_COMPAT_LIBRARY "${port}/compat.lib" PARENT_SCOPE) - add_library(moonlight_nxdk_compat STATIC "${compat}/runtime.c") + add_library(moonlight_nxdk_compat STATIC "${compat}/runtime.c" "${compat}/ffmpeg_memory.c") target_include_directories(moonlight_nxdk_compat PUBLIC "${compat}/include") target_compile_options(moonlight_nxdk_compat INTERFACE -include "${compat}/include/moonlight_posix.h") diff --git a/cmake/nxdk.cmake b/cmake/nxdk.cmake index 4553783..bef9761 100644 --- a/cmake/nxdk.cmake +++ b/cmake/nxdk.cmake @@ -344,6 +344,8 @@ function(moonlight_add_xbox_build) -DMOONLIGHT_MAKE_PROGRAM:FILEPATH=${xbox_make_program} -DMOONLIGHT_TOOLCHAIN_FILE:FILEPATH=${toolchain_file} -DMOONLIGHT_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} + -DMOONLIGHT_BUILD_VIDEO_EXPERIMENTS:BOOL=${MOONLIGHT_BUILD_VIDEO_EXPERIMENTS} + -DMOONLIGHT_ENABLE_GPU_PRESENTATION:BOOL=${MOONLIGHT_ENABLE_GPU_PRESENTATION} -DMOONLIGHT_INPUT_DIAGNOSTICS:BOOL=${MOONLIGHT_INPUT_DIAGNOSTICS} -P "${driver_script}" ) diff --git a/cmake/run-child-build.cmake b/cmake/run-child-build.cmake index 98c2bf9..a21ea26 100644 --- a/cmake/run-child-build.cmake +++ b/cmake/run-child-build.cmake @@ -28,6 +28,8 @@ if(MOONLIGHT_COMMAND_MODE STREQUAL "configure") -S "${MOONLIGHT_SOURCE_DIR}" -B "${MOONLIGHT_BINARY_DIR}" -DMOONLIGHT_BUILD_KIND:STRING=XBOX + "-DMOONLIGHT_BUILD_VIDEO_EXPERIMENTS:BOOL=${MOONLIGHT_BUILD_VIDEO_EXPERIMENTS}" + "-DMOONLIGHT_ENABLE_GPU_PRESENTATION:BOOL=${MOONLIGHT_ENABLE_GPU_PRESENTATION}" "-DMOONLIGHT_INPUT_DIAGNOSTICS:BOOL=${MOONLIGHT_INPUT_DIAGNOSTICS}" "-DMOONLIGHT_NXDK_DIR:PATH=${MOONLIGHT_NXDK_SOURCE_DIR}" "-DMOONLIGHT_NXDK_BUILD_DIR:PATH=${MOONLIGHT_NXDK_DIR}" diff --git a/cmake/xbox-build.cmake b/cmake/xbox-build.cmake index 5138610..ab83b2e 100644 --- a/cmake/xbox-build.cmake +++ b/cmake/xbox-build.cmake @@ -98,12 +98,17 @@ target_compile_options(${CMAKE_PROJECT_NAME} ${MOONLIGHT_COMPILE_OPTIONS} $<$:-std=gnu++17> ) +# NASM's freestanding i386 objects do not carry Windows SafeSEH metadata; the +# final Xbox image is packaged as XBE and does not use the Windows PE loader. +target_link_options(${CMAKE_PROJECT_NAME} PRIVATE /safeseh:no) target_compile_definitions(${CMAKE_PROJECT_NAME} PRIVATE XBOX NXDK TOML_EXCEPTIONS=0 TOML_ENABLE_WINDOWS_COMPAT=0) +target_compile_definitions(${CMAKE_PROJECT_NAME} PRIVATE + MOONLIGHT_ENABLE_GPU_PRESENTATION=$) target_compile_definitions(${CMAKE_PROJECT_NAME} PRIVATE MOONLIGHT_INPUT_DIAGNOSTICS=$) add_dependencies(${CMAKE_PROJECT_NAME} moonlight-common-c) @@ -134,3 +139,53 @@ add_custom_target(xbe_iso ALL VERBATIM ) add_dependencies(xbe_iso cxbe_convert) + +if(MOONLIGHT_BUILD_VIDEO_EXPERIMENTS) + add_executable(pvideo_presentation_probe + "${CMAKE_SOURCE_DIR}/experiments/pvideo_presentation_probe.cpp" + "${CMAKE_SOURCE_DIR}/src/streaming/stats_overlay.cpp" + "${CMAKE_SOURCE_DIR}/src/streaming/video_acceleration.cpp" + "${CMAKE_SOURCE_DIR}/src/streaming/pvideo_presenter.cpp" + "${CMAKE_SOURCE_DIR}/src/streaming/video_overlay.cpp") + target_include_directories(pvideo_presentation_probe PRIVATE "${CMAKE_SOURCE_DIR}") + target_compile_definitions(pvideo_presentation_probe PRIVATE NXDK XBOX) + target_compile_options(pvideo_presentation_probe PRIVATE -std=gnu++17) + target_link_libraries(pvideo_presentation_probe PRIVATE NXDK::NXDK NXDK::NXDK_CXX) + set(_pvideo_probe_xbe_dir "${CMAKE_CURRENT_BINARY_DIR}/pvideo-presentation-probe") + file(MAKE_DIRECTORY "${_pvideo_probe_xbe_dir}") + add_custom_target(pvideo_presentation_probe_iso ALL + COMMAND "${NXDK_DIR}/tools/cxbe/cxbe" + "-OUT:${_pvideo_probe_xbe_dir}/default.xbe" + "-TITLE:PVIDEO presentation probe" + "$" + COMMAND "${NXDK_DIR}/tools/extract-xiso/build/extract-xiso" + -c "${_pvideo_probe_xbe_dir}" "PvideoPresentationProbe.iso" + DEPENDS pvideo_presentation_probe + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + COMMENT "Package the isolated overlay presentation and teardown probe" + VERBATIM) + + add_executable(mpeg2_motion_probe + "${CMAKE_SOURCE_DIR}/experiments/mpeg2_motion_probe.cpp" + "${CMAKE_SOURCE_DIR}/src/streaming/mpeg2_prediction.cpp" + "${CMAKE_SOURCE_DIR}/src/streaming/video_acceleration.cpp" + "${CMAKE_SOURCE_DIR}/src/streaming/pvideo_presenter.cpp" + "${CMAKE_SOURCE_DIR}/src/streaming/video_overlay.cpp") + target_include_directories(mpeg2_motion_probe PRIVATE "${CMAKE_SOURCE_DIR}") + target_compile_definitions(mpeg2_motion_probe PRIVATE NXDK XBOX) + target_compile_options(mpeg2_motion_probe PRIVATE -std=gnu++17) + target_link_libraries(mpeg2_motion_probe PRIVATE NXDK::NXDK NXDK::NXDK_CXX) + set(_mpeg2_probe_xbe_dir "${CMAKE_CURRENT_BINARY_DIR}/mpeg2-motion-probe") + file(MAKE_DIRECTORY "${_mpeg2_probe_xbe_dir}") + add_custom_target(mpeg2_motion_probe_iso ALL + COMMAND "${NXDK_DIR}/tools/cxbe/cxbe" + "-OUT:${_mpeg2_probe_xbe_dir}/default.xbe" + "-TITLE:MPEG-2 motion probe" + "$" + COMMAND "${NXDK_DIR}/tools/extract-xiso/build/extract-xiso" + -c "${_mpeg2_probe_xbe_dir}" "Mpeg2MotionProbe.iso" + DEPENDS mpeg2_motion_probe + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + COMMENT "Package the isolated MPEG-2 motion prediction experiment" + VERBATIM) +endif() diff --git a/dockle.toml b/dockle.toml index d7aff06..aeadc97 100644 --- a/dockle.toml +++ b/dockle.toml @@ -20,7 +20,7 @@ source = "." home = true [targets.doxygen] -inputs = ["README.md", "src"] +inputs = ["README.md", "src", "experiments/README.md"] image_paths = ["docs/images"] predefined = ["NXDK"] extra_stylesheets = [ diff --git a/experiments/README.md b/experiments/README.md new file mode 100644 index 0000000..1c44419 --- /dev/null +++ b/experiments/README.md @@ -0,0 +1,52 @@ +# Video experiments + +These optional Xbox programs exercise video presentation and fractional sampling independently +of a streaming host. They use the production presenter and are disabled in the default build. + +## Build and run + +From the repository root, using the normal build prerequisites and toolchain: + +```bash +cmake -S . -B cmake-build-experiments -DCMAKE_BUILD_TYPE=Release -DBUILD_DOCS=OFF -DMOONLIGHT_BUILD_VIDEO_EXPERIMENTS=ON +cmake --build cmake-build-experiments +``` + +On Windows, use an MSYS2 MinGW64 shell and add `-G "MinGW Makefiles"` to the configure command. +The Xbox child build selects Unix Makefiles automatically. The generated files are: + +| Probe | ISO | XBE | +| ----------------- | ---------------------------------------------------------- | -------------------------------------------------------------------- | +| Presentation | `cmake-build-experiments/xbox/PvideoPresentationProbe.iso` | `cmake-build-experiments/xbox/pvideo-presentation-probe/default.xbe` | +| MPEG-2 prediction | `cmake-build-experiments/xbox/Mpeg2MotionProbe.iso` | `cmake-build-experiments/xbox/mpeg2-motion-probe/default.xbe` | + +Launch an ISO with the repository's configured xemu installation, for example: + +```bash +./scripts/run-xemu.sh --iso cmake-build-experiments/xbox/PvideoPresentationProbe.iso +``` + +From a Windows command prompt, use `scripts\run-xemu.cmd --iso` with the same ISO path. +The probes use GPU presentation directly; `MOONLIGHT_ENABLE_GPU_PRESENTATION` controls the +streaming client independently. + +## Presentation probe + +The probe checks numeric statistics formatting, displays a gray CPU framebuffer, then runs two +overlay sessions. Each shows rotating red, green, blue and white bars scaled from an HD source. +After both sessions, **PVIDEO PROBE COMPLETE** appears on the restored CPU framebuffer. + +Inspect the colors, scaling, buffer updates and framebuffer restoration. Submission or teardown +failures display an error. In xemu, use a graphical display: display updates drive its emulated +vertical blank, so a headless run cannot validate the presentation path. + +## MPEG-2 prediction probe + +The probe cycles integer and half-pixel motion offsets. A CPU-computed luma reference appears on +the left, and GPU sampling appears on the right. Compare the two while checking the displayed +motion vector. The sequence repeats until the program is closed. + +This is a visual comparison of one prediction stage, not a compressed-video decoder or a +bit-exact reconstruction test. The overlay is composed during scanout and cannot be read back +through the RGB framebuffer. Console testing is needed to assess image correctness and +performance on hardware; emulator output alone cannot establish those properties. diff --git a/experiments/mpeg2_motion_probe.cpp b/experiments/mpeg2_motion_probe.cpp new file mode 100644 index 0000000..befce54 --- /dev/null +++ b/experiments/mpeg2_motion_probe.cpp @@ -0,0 +1,84 @@ +/** + * @file experiments/mpeg2_motion_probe.cpp + * @brief Compares CPU MPEG-2 half-pixel predictions with NV2A overlay sampling on a console. + */ +#include "src/streaming/mpeg2_prediction.h" +#include "src/streaming/pvideo_presenter.h" + +#include +#include +#include +#include +#include +#include + +namespace { + constexpr int SOURCE_SIZE = 144; ///< Reference plane includes a border for negative motion vectors. + constexpr int BLOCK_SIZE = 128; ///< An eight-by-eight grid of 16-pixel luma blocks. + + /** + * @brief Draw CPU-computed reference values into the separate GPU scanout surface. + * + * @param prediction Packed reference predictions. + */ + void draw_cpu_prediction(const std::array &prediction) { + for (int row = 0; row < BLOCK_SIZE; ++row) { + for (int column = 0; column < BLOCK_SIZE; ++column) { + const auto gray = static_cast((prediction[row * BLOCK_SIZE + column] - 16) * 255 + 109) / 219U; + pb_fill(64 + column, 100 + row, 1, 1, 0xff000000U | gray * 0x010101U); + } + } + } +} // namespace + +/** + * @brief Cycle integer and half-pixel motion cases for visual comparison on hardware. + * + * The overlay is composed during scanout and cannot be read back through the RGB + * framebuffer. This experiment reports submission errors and displays the CPU + * oracle alongside it; it does not certify bit-exact GPU reconstruction. + * @return Nonzero if the output mode or overlay could not be initialized. + */ +int main() { + if (!XVideoSetMode(640, 480, 32, REFRESH_DEFAULT)) { + return 1; + } + static std::array luma {}; + static std::array chroma {}; + static std::array prediction {}; + chroma.fill(128); + for (int row = 0; row < SOURCE_SIZE; ++row) { + for (int column = 0; column < SOURCE_SIZE; ++column) { + luma[row * SOURCE_SIZE + column] = static_cast(16 + ((column * 17 + row * 29) % 220)); + } + } + const streaming::Yuv420FrameView frame {SOURCE_SIZE, SOURCE_SIZE, {luma.data(), chroma.data(), chroma.data()}, {luma.size(), chroma.size(), chroma.size()}, {SOURCE_SIZE, SOURCE_SIZE / 2, SOURCE_SIZE / 2}}; + const streaming::Mpeg2LumaPlane reference {luma.data(), luma.size(), SOURCE_SIZE, SOURCE_SIZE, SOURCE_SIZE}; + streaming::PvideoPresenter presenter; + for (;;) { + for (int motionY = -1; motionY <= 1; ++motionY) { + for (int motionX = -1; motionX <= 1; ++motionX) { + presenter.reset(); + std::memset(XVideoGetFB(), 0, 640 * 480 * 4); + debugPrint("MPEG-2 prediction experiment\nCPU reference left | GPU sampling right\nMotion: %d/2, %d/2 pixels\nVisual comparison only; not a hardware decoder.\n", motionX, motionY); + if (!streaming::predict_mpeg2_luma(reference, 16 + motionX, 16 + motionY, BLOCK_SIZE, BLOCK_SIZE, prediction.data(), prediction.size(), BLOCK_SIZE)) { + return 2; + } + std::string error; + if (!presenter.present(frame, {352, 100, BLOCK_SIZE, BLOCK_SIZE}, &error, {16 + motionX, 16 + motionY, BLOCK_SIZE, BLOCK_SIZE})) { + debugPrint("GPU EXPERIMENT FAILED: %s\n", error.c_str()); + Sleep(10000); + return 3; + } + // xemu composes PVIDEO only over a GPU-rendered scanout surface. + // Draw the CPU oracle there as RGB; its prediction arithmetic and + // range conversion remain independent of PVIDEO's YUV conversion. + draw_cpu_prediction(prediction); + pb_erase_text_screen(); + pb_print("MPEG-2 prediction experiment\nCPU reference left | GPU sampling right\nMotion: %d/2, %d/2 pixels\nVisual comparison only; not a hardware decoder.\n", motionX, motionY); + pb_draw_text_screen(); + Sleep(5000); + } + } + } +} diff --git a/experiments/pvideo_presentation_probe.cpp b/experiments/pvideo_presentation_probe.cpp new file mode 100644 index 0000000..087a6bd --- /dev/null +++ b/experiments/pvideo_presentation_probe.cpp @@ -0,0 +1,112 @@ +/** + * @file experiments/pvideo_presentation_probe.cpp + * @brief Exercises GPU overlay colors, buffer updates and restoration independently of networking and input. + */ +#include "src/streaming/pvideo_presenter.h" +#include "src/streaming/stats_overlay.h" + +#include +#include +#include +#include +#include + +namespace { + constexpr int WIDTH = 1280; ///< HD source width exercises full-size DMA buffer rotation. + constexpr int HEIGHT = 720; ///< HD source height exercises downscaling to the probe output. + + /** + * @brief Fill limited-range BT.601 red, green, blue and white bars. + * + * @param y Luma storage. + * @param u Blue-difference chroma storage. + * @param v Red-difference chroma storage. + * @param phase Rotation of the four color bars. + */ + void fill_bars(std::array &y, std::array &u, std::array &v, int phase) { + constexpr std::array, 4> colors {{{81, 90, 240}, {145, 54, 34}, {41, 240, 110}, {235, 128, 128}}}; + for (int row = 0; row < HEIGHT; ++row) { + for (int column = 0; column < WIDTH; ++column) { + y[row * WIDTH + column] = colors[(column / (WIDTH / 4) + phase) % 4][0]; + } + } + for (int row = 0; row < HEIGHT / 2; ++row) { + for (int column = 0; column < WIDTH / 2; ++column) { + const auto &color = colors[(column / (WIDTH / 8) + phase) % 4]; + u[row * (WIDTH / 2) + column] = color[1]; + v[row * (WIDTH / 2) + column] = color[2]; + } + } + } +} // namespace + +/** + * @brief Run two overlay sessions with visible color changes, then leave the restored framebuffer visible. + * + * @return Nonzero if mode setup, overlay submission or teardown fails. + */ +int main() { + if (!XVideoSetMode(640, 480, 32, REFRESH_DEFAULT)) { + return 1; + } + std::memset(XVideoGetFB(), 0x40, 640 * 480 * 4); + debugResetCursor(); + streaming::StreamStatisticsSnapshot statistics {}; + statistics.width = 1280; + statistics.height = 720; + statistics.fps = 30; + statistics.incomingFps = 29.96; + statistics.decodingFps = 18.24; + statistics.renderingFps = 18.0; + statistics.decodeTimeMs = 41.35; + statistics.presentationTimeMs = 2.5; + statistics.videoBitrateMbps = 4.2; + statistics.hostProcessingMs = 6.8; + statistics.presentationPath = "GPU"; + statistics.videoCodec = "H.264"; + statistics.decoderFeatures = "MMX/SSE"; + const auto statisticLines = streaming::build_live_stats_overlay_lines(statistics); + if (statisticLines[1] != "FPS: incoming 30.0 | decoded 18.2 | rendered 18.0") { + debugPrint("STATISTICS FORMAT FAILED\n"); + for (;;) { + Sleep(1000); + } + } + debugPrint("STATISTICS NUMERIC FORMAT PASSED\n%s\n", statisticLines[1].c_str()); + debugPrint("CPU FRAMEBUFFER BASELINE\nGPU test begins after 2 seconds.\n"); + Sleep(2000); + static std::array y {}; + static std::array u {}; + static std::array v {}; + const streaming::Yuv420FrameView frame {WIDTH, HEIGHT, {y.data(), u.data(), v.data()}, {y.size(), u.size(), v.size()}, {WIDTH, WIDTH / 2, WIDTH / 2}}; + streaming::PvideoPresenter presenter; + for (int session = 0; session < 2; ++session) { + for (int number = 0; number < 180; ++number) { + fill_bars(y, u, v, number / 30); + std::string error; + if (!presenter.present(frame, {64, 144, 512, 288}, &error)) { + debugResetCursor(); + debugPrint("PVIDEO PROBE FAILED\n%s\n", error.c_str()); + Sleep(20000); + return 2; + } + Sleep(33); + } + if (!presenter.reset()) { + debugResetCursor(); + debugPrint("PVIDEO TEARDOWN FAILED\n"); + Sleep(20000); + return 3; + } + std::memset(XVideoGetFB(), 0x30, 640 * 480 * 4); + debugResetCursor(); + debugPrint("Overlay stopped; CPU framebuffer restored.\nSession %d completed.\n", session + 1); + Sleep(2000); + } + std::memset(XVideoGetFB(), 0x30, 640 * 480 * 4); + debugResetCursor(); + debugPrint("PVIDEO PROBE COMPLETE\nBoth overlay sessions stopped and restored the framebuffer.\n"); + for (;;) { + Sleep(1000); + } +} diff --git a/scripts/ffmpeg-nxdk-nasm.sh b/scripts/ffmpeg-nxdk-nasm.sh new file mode 100755 index 0000000..b1a1115 --- /dev/null +++ b/scripts/ffmpeg-nxdk-nasm.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +set -eu + +# FFmpeg's freestanding target defaults to ELF. Xbox libraries require i386 COFF. +# A final format option overrides configure's default without changing its probes. +exec nasm "$@" -f win32 diff --git a/src/_nxdk_compat/ffmpeg_compat.h b/src/_nxdk_compat/ffmpeg_compat.h index 52d3eec..0911e27 100644 --- a/src/_nxdk_compat/ffmpeg_compat.h +++ b/src/_nxdk_compat/ffmpeg_compat.h @@ -6,10 +6,23 @@ * descriptors, mapped files and temporary files are outside this decoder build. */ #pragma once +#include "ffmpeg_memory.h" + #include +#include #include #include +/** @brief Supply the compiler memory fence required by FFmpeg's Win32 thread probe. */ +#define MemoryBarrier() __atomic_thread_fence(__ATOMIC_SEQ_CST) + +/** @brief Route FFmpeg aligned allocation to a complete private implementation. */ +#define _aligned_malloc moonlight_ffmpeg_aligned_malloc +/** @brief Preserve aligned buffers when FFmpeg reallocates them. */ +#define _aligned_realloc moonlight_ffmpeg_aligned_realloc +/** @brief Match the private aligned allocator when releasing FFmpeg buffers. */ +#define _aligned_free moonlight_ffmpeg_aligned_free + /** * @brief Reject an optional FFmpeg open request. * diff --git a/src/_nxdk_compat/ffmpeg_memory.c b/src/_nxdk_compat/ffmpeg_memory.c new file mode 100644 index 0000000..dbc5bf4 --- /dev/null +++ b/src/_nxdk_compat/ffmpeg_memory.c @@ -0,0 +1,60 @@ +/** + * @file src/_nxdk_compat/ffmpeg_memory.c + * @brief Implements aligned FFmpeg buffers without nxdk's incomplete CRT allocator. + */ +#include "ffmpeg_memory.h" + +#include +#include +#include +#include + +/** + * @brief Store the owning allocation and logical size immediately before aligned data. + */ +typedef struct MoonlightAlignedAllocation { + void *allocation; ///< Pointer returned by malloc. + size_t size; ///< Requested byte count, used to bound reallocation copies. +} MoonlightAlignedAllocation; + +void *moonlight_ffmpeg_aligned_malloc(size_t size, size_t alignment) { + if (alignment < sizeof(void *) || (alignment & (alignment - 1)) != 0) { + errno = EINVAL; + return NULL; + } + const size_t overhead = sizeof(MoonlightAlignedAllocation) + alignment - 1; + if (overhead < alignment || size > SIZE_MAX - overhead) { + errno = ENOMEM; + return NULL; + } + void *allocation = malloc(size + overhead); + if (allocation == NULL) { + errno = ENOMEM; + return NULL; + } + const uintptr_t address = ((uintptr_t) allocation + sizeof(MoonlightAlignedAllocation) + alignment - 1) & ~(uintptr_t) (alignment - 1); + MoonlightAlignedAllocation *header = (MoonlightAlignedAllocation *) address - 1; + header->allocation = allocation; + header->size = size; + return (void *) address; +} + +void moonlight_ffmpeg_aligned_free(void *pointer) { + if (pointer != NULL) { + free(((MoonlightAlignedAllocation *) pointer - 1)->allocation); + } +} + +void *moonlight_ffmpeg_aligned_realloc(void *pointer, size_t size, size_t alignment) { + if (size == 0) { + moonlight_ffmpeg_aligned_free(pointer); + return NULL; + } + void *replacement = moonlight_ffmpeg_aligned_malloc(size, alignment); + if (replacement != NULL && pointer != NULL) { + const size_t previousSize = ((MoonlightAlignedAllocation *) pointer - 1)->size; + memcpy(replacement, pointer, previousSize < size ? previousSize : size); + moonlight_ffmpeg_aligned_free(pointer); + } + return replacement; +} diff --git a/src/_nxdk_compat/ffmpeg_memory.h b/src/_nxdk_compat/ffmpeg_memory.h new file mode 100644 index 0000000..22be585 --- /dev/null +++ b/src/_nxdk_compat/ffmpeg_memory.h @@ -0,0 +1,43 @@ +/** + * @file src/_nxdk_compat/ffmpeg_memory.h + * @brief Declares the aligned allocation family used privately by FFmpeg. + */ +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + + /** + * @brief Allocate memory with a power-of-two alignment. + * + * @param size Requested byte count. + * @param alignment Alignment, at least the alignment of a pointer. + * + * @return Aligned allocation, or NULL with errno set on failure. + */ + void *moonlight_ffmpeg_aligned_malloc(size_t size, size_t alignment); + + /** + * @brief Resize an aligned allocation while preserving its contents on success. + * + * @param pointer Previous allocation, or NULL. + * @param size New byte count; zero releases the allocation. + * @param alignment Requested power-of-two alignment. + * + * @return Replacement allocation, or NULL; failure preserves the old allocation. + */ + void *moonlight_ffmpeg_aligned_realloc(void *pointer, size_t size, size_t alignment); + + /** + * @brief Release memory allocated by the private aligned allocation family. + * + * @param pointer Allocation to release, or NULL. + */ + void moonlight_ffmpeg_aligned_free(void *pointer); + +#ifdef __cplusplus +} +#endif diff --git a/src/app/client_state.cpp b/src/app/client_state.cpp index f164044..c0e2020 100644 --- a/src/app/client_state.cpp +++ b/src/app/client_state.cpp @@ -558,8 +558,8 @@ namespace { }, { "toggle-show-performance-stats", - std::string("Show End Stream Stats: ") + (state.settings.showPerformanceStats ? "On" : "Off"), - "Toggle the performance summary shown after streaming ends.", + std::string("Performance Stats: ") + (state.settings.showPerformanceStats ? "On" : "Off"), + "Show performance statistics during playback and after the stream ends.", true, }, }; @@ -2016,7 +2016,7 @@ namespace app { state.settings.showPerformanceStats = !state.settings.showPerformanceStats; state.settings.dirty = true; update->persistence.settingsChanged = true; - state.shell.statusMessage = std::string("End stream performance stats ") + (state.settings.showPerformanceStats ? "enabled" : "disabled"); + state.shell.statusMessage = std::string("Performance stats ") + (state.settings.showPerformanceStats ? "enabled" : "disabled"); rebuild_menu(state, "toggle-show-performance-stats"); return; } diff --git a/src/app/client_state.h b/src/app/client_state.h index 323de80..e3e87cf 100644 --- a/src/app/client_state.h +++ b/src/app/client_state.h @@ -322,7 +322,7 @@ namespace app { int streamFramerate = 30; ///< Preferred stream frame rate in frames per second. int streamBitrateKbps = 1000; ///< Preferred stream bitrate in kilobits per second. bool playAudioOnPc = false; ///< True when the host PC should continue local audio playback during streaming. - bool showPerformanceStats = false; ///< True when stream telemetry should be shown after streaming ends. + bool showPerformanceStats = false; ///< Display live performance telemetry and include it in the stream summary. bool playAudioOnXbox = true; ///< True when the Xbox should decode and play streamed audio locally. bool dirty = false; ///< True when persisted TOML-backed settings changed and should be saved. std::vector savedFiles; ///< Saved-file catalog shown on the reset settings page. diff --git a/src/app/settings_storage.h b/src/app/settings_storage.h index c322b6f..e0a5334 100644 --- a/src/app/settings_storage.h +++ b/src/app/settings_storage.h @@ -25,7 +25,7 @@ namespace app { int streamFramerate = 30; ///< Preferred stream frame rate in frames per second. int streamBitrateKbps = 1000; ///< Preferred stream bitrate in kilobits per second. bool playAudioOnPc = false; ///< True when the host PC should continue local audio playback during streaming. - bool showPerformanceStats = false; ///< True when stream telemetry should be shown after streaming ends. + bool showPerformanceStats = false; ///< Display live performance telemetry and include it in the stream summary. bool playAudioOnXbox = true; ///< True when the Xbox should decode and play streamed audio locally. }; diff --git a/src/streaming/aligned_media_buffer.h b/src/streaming/aligned_media_buffer.h new file mode 100644 index 0000000..5364556 --- /dev/null +++ b/src/streaming/aligned_media_buffer.h @@ -0,0 +1,105 @@ +/** + * @file src/streaming/aligned_media_buffer.h + * @brief Provides aligned, padded storage for buffers passed to FFmpeg SIMD routines. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace streaming { + /** + * @brief Owns byte storage with a 64-byte-aligned start and zeroed read padding. + */ + class AlignedMediaBuffer { + public: + static constexpr std::size_t PADDING = 64; ///< FFmpeg's required input overread padding. + + /** + * @brief Construct an empty media buffer. + */ + AlignedMediaBuffer() = default; + AlignedMediaBuffer(const AlignedMediaBuffer &) = delete; + AlignedMediaBuffer &operator=(const AlignedMediaBuffer &) = delete; + + /** + * @brief Transfer an aligned allocation without relocating its payload. + * + * @param other Buffer relinquishing its allocation. + */ + AlignedMediaBuffer(AlignedMediaBuffer &&other) noexcept: + storage_(std::move(other.storage_)), + size_(std::exchange(other.size_, 0)) { + } + + /** + * @brief Replace this allocation with another buffer's aligned storage. + * + * @param other Buffer relinquishing its allocation. + * + * @return This buffer after ownership transfer. + */ + AlignedMediaBuffer &operator=(AlignedMediaBuffer &&other) noexcept { + if (this != &other) { + storage_ = std::move(other.storage_); + size_ = std::exchange(other.size_, 0); + } + return *this; + } + + /** + * @brief Resize storage, discarding prior payload contents and zeroing trailing read padding. + * + * @param size Required payload bytes. + */ + void resize(std::size_t size) { + if (size > std::numeric_limits::max() - PADDING - 63) { + std::abort(); + } + storage_.resize(size + PADDING + 63); + size_ = size; + std::memset(data() + size_, 0, PADDING); + } + + /** + * @brief Return the aligned payload address. + * @return Payload storage, or nullptr for an empty buffer. + */ + std::uint8_t *data() { + return storage_.empty() ? nullptr : reinterpret_cast((reinterpret_cast(storage_.data()) + 63U) & ~std::uintptr_t {63}); + } + + /** + * @brief Return the aligned read-only payload address. + * @return Payload storage, or nullptr for an empty buffer. + */ + const std::uint8_t *data() const { + return storage_.empty() ? nullptr : reinterpret_cast((reinterpret_cast(storage_.data()) + 63U) & ~std::uintptr_t {63}); + } + + /** + * @brief Return the payload size excluding padding. + * @return Usable byte count. + */ + std::size_t size() const { + return size_; + } + + /** + * @brief Clear the payload while retaining reusable allocation capacity. + */ + void clear() { + storage_.clear(); + size_ = 0; + } + + private: + std::vector storage_; ///< Overallocated backing storage. + std::size_t size_ = 0; ///< Payload length before the zero padding. + }; +} // namespace streaming diff --git a/src/streaming/ffmpeg_stream_backend.cpp b/src/streaming/ffmpeg_stream_backend.cpp index 7ebf4cc..c4c68af 100644 --- a/src/streaming/ffmpeg_stream_backend.cpp +++ b/src/streaming/ffmpeg_stream_backend.cpp @@ -18,6 +18,7 @@ extern "C" { #include #include +#include #include #include #include @@ -41,6 +42,7 @@ extern "C" { * @return Microseconds from the platform timer used by moonlight-common-c. */ extern "C" std::uint64_t PltGetMicroseconds(void); +static_assert(streaming::AlignedMediaBuffer::PADDING >= AV_INPUT_BUFFER_PADDING_SIZE, "Media buffers must retain FFmpeg's SIMD read padding"); namespace { @@ -334,6 +336,27 @@ namespace { namespace streaming { + VideoAccelerationReport FfmpegStreamBackend::acceleration_report() const { + return acceleration_; + } + + VideoStatisticsCounters FfmpegStreamBackend::performance_counters() const { + return statistics_.counters(); + } + + std::string FfmpegStreamBackend::presentation_path() const { + return presenter_.active() ? "GPU" : (acceleration_.softwareFrames > 0 ? "Software" : "Waiting"); + } + + void FfmpegStreamBackend::set_statistics_bitmap(int width, int height, std::vector grayscale) { + statisticsOverlay_.set_bitmap(width, height, std::move(grayscale)); + statisticsBitmapChanged_ = statisticsTextureChanged_ = true; + if (statisticsTexture_ != nullptr) { + SDL_DestroyTexture(statisticsTexture_); + statisticsTexture_ = nullptr; + } + } + FfmpegStreamBackend::~FfmpegStreamBackend() { shutdown(); } @@ -378,6 +401,10 @@ namespace streaming { return video_.hasFrame.load(); } + std::string FfmpegStreamBackend::active_video_decoder_name() const { + return video_.codecName; + } + /** * @brief Return whether two SDL rectangles describe the same area. * @@ -451,6 +478,19 @@ namespace streaming { } bool FfmpegStreamBackend::render_latest_video_frame(SDL_Renderer *renderer, int screenWidth, int screenHeight, bool allowDirectFramebuffer) { + const auto before = acceleration_.gpuFrames + acceleration_.softwareFrames; + const auto started = PltGetMicroseconds(); + const bool result = render_video_frame(renderer, screenWidth, screenHeight, allowDirectFramebuffer); + if (result) { + statisticsBitmapChanged_ = false; + if (acceleration_.gpuFrames + acceleration_.softwareFrames > before) { + statistics_.presented(PltGetMicroseconds() - started); + } + } + return result; + } + + bool FfmpegStreamBackend::render_video_frame(SDL_Renderer *renderer, int screenWidth, int screenHeight, bool allowDirectFramebuffer) { if (renderer == nullptr || !video_.hasFrame.load()) { return false; } @@ -459,6 +499,9 @@ namespace streaming { if (const std::uint64_t publishedFrameVersion = video_.publishedFrameVersion.load(); publishedFrameVersion != video_.renderedFrameVersion) { std::scoped_lock lock(video_.frame_mutex()); if (video_.latestFrameVersion != video_.renderedFrameVersion && video_.latestFrame.width > 0 && video_.latestFrame.height > 0) { + if (video_.latestFrameVersion > video_.renderedFrameVersion + 1) { + statistics_.dropped(video_.latestFrameVersion - video_.renderedFrameVersion - 1); + } std::swap(video_.renderFrame, video_.latestFrame); video_.renderedFrameVersion = video_.latestFrameVersion; textureNeedsUpload = true; @@ -470,7 +513,44 @@ namespace streaming { } #ifdef NXDK + if (allowDirectFramebuffer && !overlayFailed_) { + if (!textureNeedsUpload && !statisticsBitmapChanged_ && presenter_.active()) { + return true; + } + const LatestVideoFrame &frame = video_.renderFrame; + const Yuv420FrameView view { + frame.width, + frame.height, + {frame.yPlane.data(), frame.uPlane.data(), frame.vPlane.data()}, + {frame.yPlane.size(), frame.uPlane.size(), frame.vPlane.size()}, + {frame.yPitch, frame.uPitch, frame.vPitch}, + frame.matrix, + frame.fullRange, + frame.interlaced, + }; + const SDL_Rect destination = build_letterboxed_destination(screenWidth, screenHeight, frame.width, frame.height); + if (!presenter_.active()) { + const VIDEO_MODE mode = XVideoGetMode(); + if (auto *framebuffer = XVideoGetFB(); framebuffer != nullptr && mode.width > 0 && mode.height > 0 && (mode.bpp == 16 || mode.bpp == 32)) { + std::memset(framebuffer, 0, static_cast(mode.width) * static_cast(mode.height) * static_cast(mode.bpp / 8)); + XVideoFlushFB(); + } + } + std::string reason; + if (presenter_.present(view, {destination.x, destination.y, destination.w, destination.h}, &reason, {}, &statisticsOverlay_)) { + if (textureNeedsUpload) { + ++acceleration_.gpuFrames; + } + return true; + } + overlayFailed_ = true; + acceleration_.record_fallback(reason); + logging::warn("stream", "GPU presentation fallback: " + reason); + } if (allowDirectFramebuffer && render_latest_video_frame_to_framebuffer(screenWidth, screenHeight)) { + if (textureNeedsUpload) { + ++acceleration_.softwareFrames; + } return true; } #else @@ -500,7 +580,38 @@ namespace streaming { } const SDL_Rect destination = build_letterboxed_destination(screenWidth, screenHeight, video_.renderFrame.width, video_.renderFrame.height); - return SDL_RenderCopy(renderer, video_.texture, nullptr, &destination) == 0; + const bool presented = SDL_RenderCopy(renderer, video_.texture, nullptr, &destination) == 0; + if (presented) { + render_sdl_statistics(renderer, destination); + } + if (presented && textureNeedsUpload) { + ++acceleration_.softwareFrames; + } + return presented; + } + + void FfmpegStreamBackend::render_sdl_statistics(SDL_Renderer *renderer, const SDL_Rect &destination) { + const int width = statisticsOverlay_.width(); + const int height = statisticsOverlay_.height(); + if (width == 0 || height == 0) { + return; + } + if (statisticsTexture_ == nullptr) { + statisticsTexture_ = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_ARGB8888, SDL_TEXTUREACCESS_STREAMING, width, height); + statisticsTextureChanged_ = true; + } + if (statisticsTexture_ == nullptr) { + return; + } + if (statisticsTextureChanged_) { + const auto &pixels = statisticsOverlay_.argb_pixels(); + if (SDL_UpdateTexture(statisticsTexture_, nullptr, pixels.data(), width * 4) != 0) { + return; + } + statisticsTextureChanged_ = false; + } + const SDL_Rect panel {destination.x + VideoOverlay::MARGIN, destination.y + VideoOverlay::MARGIN, width, height}; + SDL_RenderCopy(renderer, statisticsTexture_, nullptr, &panel); } int FfmpegStreamBackend::setup_video_decoder(int videoFormat, int width, int height, int redrawRate, int drFlags) { @@ -510,6 +621,16 @@ namespace streaming { (void) drFlags; cleanup_video_decoder(); + acceleration_ = {}; + statistics_.reset(); + overlayFailed_ = false; +#if defined(NXDK) && !MOONLIGHT_ENABLE_GPU_PRESENTATION + overlayFailed_ = true; + acceleration_.record_fallback("GPU presentation disabled in this build"); +#endif + const int cpuFlags = av_get_cpu_flags(); + acceleration_.cpuFeatures = (cpuFlags & AV_CPU_FLAG_SSE) != 0 ? "MMX/SSE" : ((cpuFlags & AV_CPU_FLAG_MMX) != 0 ? "MMX" : "Scalar"); + acceleration_.threaded = std::string(avcodec_configuration()).find("--enable-w32threads") != std::string::npos; if ((videoFormat & VIDEO_FORMAT_MASK_H264) == 0) { logging::error("stream", "The FFmpeg backend currently supports only H.264 video streams"); @@ -522,6 +643,7 @@ namespace streaming { return -1; } + video_.codecName = "H.264"; video_.codecContext = avcodec_alloc_context3(codec); video_.decodedFrame = av_frame_alloc(); video_.convertedFrame = av_frame_alloc(); @@ -532,6 +654,8 @@ namespace streaming { return -1; } + // One decoder worker avoids frame-threading latency on the single-core CPU. + // FFmpeg's w32threads support still provides its internal synchronization. video_.codecContext->thread_count = 1; video_.codecContext->thread_type = 0; video_.codecContext->flags |= AV_CODEC_FLAG_LOW_DELAY | AV_CODEC_FLAG_OUTPUT_CORRUPT; @@ -540,11 +664,12 @@ namespace streaming { video_.codecContext->skip_loop_filter = AVDISCARD_ALL; video_.codecContext->skip_idct = AVDISCARD_NONREF; if (const int openResult = avcodec_open2(video_.codecContext, codec, nullptr); openResult < 0) { - logging::error("stream", std::string("avcodec_open2 failed for H.264: ") + describe_ffmpeg_error(openResult)); + logging::error("stream", std::string("avcodec_open2 failed for ") + video_.codecName + ": " + describe_ffmpeg_error(openResult)); cleanup_video_decoder(); return openResult; } + logging::info("stream", std::string("Using FFmpeg ") + video_.codecName + " software decoding"); return 0; } @@ -575,8 +700,21 @@ namespace streaming { video_.decoderThread = nullptr; } + void FfmpegStreamBackend::stop_video_presentation() { + if (!presenter_.reset()) { + acceleration_.record_fallback("GPU overlay teardown timed out; acceleration disabled"); + logging::warn("stream", "GPU overlay teardown timed out; buffers retained to avoid DMA into freed storage"); + } + } + void FfmpegStreamBackend::cleanup_video_decoder() { stop_video_decoder(); + stop_video_presentation(); + if (statisticsTexture_ != nullptr) { + SDL_DestroyTexture(statisticsTexture_); + statisticsTexture_ = nullptr; + } + statisticsOverlay_.set_bitmap(0, 0, {}); if (video_.texture != nullptr) { SDL_DestroyTexture(video_.texture); @@ -617,6 +755,7 @@ namespace streaming { } video_.renderedFrameVersion = 0; video_.publishedFrameVersion.store(0); + video_.codecName = "video"; video_.directFramebufferDestination = SDL_Rect {0, 0, 0, 0}; video_.directFramebufferCleared = false; video_.decoderStopRequested.store(false); @@ -646,12 +785,18 @@ namespace streaming { if (!LiWaitForNextVideoFrame(&frameHandle, &decodeUnit)) { break; } + record_received_unit(decodeUnit); int decodeStatus = DR_OK; if (!video_.decoderStopRequested.load() && decodeUnit != nullptr) { decodeStatus = prepare_video_decode_unit(&frameHandle, &decodeUnit); if (decodeStatus == DR_OK) { + const auto started = PltGetMicroseconds(); decodeStatus = decode_video_decode_unit(decodeUnit); + statistics_.decode_time(PltGetMicroseconds() - started, video_.lastDecodeQueueUs.load()); + } + if (decodeStatus != DR_OK) { + statistics_.dropped(1); } } LiCompleteVideoFrame(frameHandle, decodeStatus); @@ -663,6 +808,12 @@ namespace streaming { return 0; } + void FfmpegStreamBackend::record_received_unit(PDECODE_UNIT unit) { + if (unit != nullptr && unit->fullLength > 0) { + statistics_.received(static_cast(unit->frameNumber), static_cast(unit->fullLength), unit->frameHostProcessingLatency); + } + } + int FfmpegStreamBackend::drop_queued_video_decode_units(VIDEO_FRAME_HANDLE *frameHandle, PDECODE_UNIT *decodeUnit) { if (frameHandle == nullptr || decodeUnit == nullptr || *frameHandle == nullptr || *decodeUnit == nullptr) { return 0; @@ -678,6 +829,7 @@ namespace streaming { if (!LiPollNextVideoFrame(&nextHandle, &nextDecodeUnit)) { break; } + record_received_unit(nextDecodeUnit); LiCompleteVideoFrame(newestHandle, STREAM_VIDEO_DROPPED_FRAME_STATUS); newestHandle = nextHandle; @@ -691,6 +843,7 @@ namespace streaming { *frameHandle = newestHandle; *decodeUnit = newestDecodeUnit; + statistics_.dropped(static_cast(droppedFrames)); if (const std::uint64_t droppedDecodeUnitCount = video_.droppedDecodeUnitCount.fetch_add(static_cast(droppedFrames)) + static_cast(droppedFrames); droppedDecodeUnitCount <= static_cast(droppedFrames) || (droppedDecodeUnitCount % STREAM_VIDEO_DROP_LOG_INTERVAL) < static_cast(droppedFrames)) { logging::warn( @@ -728,6 +881,21 @@ namespace streaming { nextFrame.yPitch = frameToPresent->linesize[0]; nextFrame.uPitch = frameToPresent->linesize[1]; nextFrame.vPitch = frameToPresent->linesize[2]; + nextFrame.fullRange = frameToPresent->color_range == AVCOL_RANGE_JPEG; + nextFrame.interlaced = (frameToPresent->flags & AV_FRAME_FLAG_INTERLACED) != 0; + switch (frameToPresent->colorspace) { + case AVCOL_SPC_BT709: + nextFrame.matrix = VideoColorMatrix::bt709; + break; + case AVCOL_SPC_UNSPECIFIED: + case AVCOL_SPC_BT470BG: + case AVCOL_SPC_SMPTE170M: + nextFrame.matrix = VideoColorMatrix::bt601; + break; + default: + nextFrame.matrix = VideoColorMatrix::unsupported; + break; + } const int chromaHeight = (frameToPresent->height + 1) / 2; const std::size_t yPlaneSize = static_cast(frameToPresent->linesize[0]) * static_cast(frameToPresent->height); @@ -751,6 +919,7 @@ namespace streaming { video_.lastDecodedFrameUs.store(PltGetMicroseconds()); video_.hasFrame.store(true); video_.decodedFrameCount.fetch_add(1); + statistics_.decoded(); return true; } @@ -826,6 +995,7 @@ namespace streaming { return false; } + statisticsOverlay_.draw_rgb(framebuffer, static_cast(framebufferPitch) * videoMode.height, framebufferPitch, videoMode.width, videoMode.height, videoMode.bpp, {destination.x, destination.y, destination.w, destination.h}); XVideoFlushFB(); return true; #else @@ -842,7 +1012,7 @@ namespace streaming { return receiveResult; } if (receiveResult < 0) { - logging::warn("stream", std::string("avcodec_receive_frame failed for H.264: ") + describe_ffmpeg_error(receiveResult)); + logging::warn("stream", std::string("avcodec_receive_frame failed for ") + video_.codecName + ": " + describe_ffmpeg_error(receiveResult)); return receiveResult; } @@ -867,7 +1037,7 @@ namespace streaming { return AVERROR(EINVAL); } - const int bufferSize = av_image_get_buffer_size(AV_PIX_FMT_YUV420P, video_.decodedFrame->width, video_.decodedFrame->height, 1); + const int bufferSize = av_image_get_buffer_size(AV_PIX_FMT_YUV420P, video_.decodedFrame->width, video_.decodedFrame->height, 32); if (bufferSize <= 0) { logging::warn("stream", "av_image_get_buffer_size failed for converted video frame"); av_frame_unref(video_.decodedFrame); @@ -879,7 +1049,10 @@ namespace streaming { video_.convertedFrame->format = AV_PIX_FMT_YUV420P; video_.convertedFrame->width = video_.decodedFrame->width; video_.convertedFrame->height = video_.decodedFrame->height; - if (const int fillResult = av_image_fill_arrays(video_.convertedFrame->data, video_.convertedFrame->linesize, video_.convertedBuffer.data(), AV_PIX_FMT_YUV420P, video_.decodedFrame->width, video_.decodedFrame->height, 1); fillResult < 0) { + video_.convertedFrame->color_range = video_.decodedFrame->color_range; + video_.convertedFrame->colorspace = video_.decodedFrame->colorspace; + video_.convertedFrame->flags = video_.decodedFrame->flags; + if (const int fillResult = av_image_fill_arrays(video_.convertedFrame->data, video_.convertedFrame->linesize, video_.convertedBuffer.data(), AV_PIX_FMT_YUV420P, video_.decodedFrame->width, video_.decodedFrame->height, 32); fillResult < 0) { logging::warn("stream", std::string("av_image_fill_arrays failed for converted frame: ") + describe_ffmpeg_error(fillResult)); av_frame_unref(video_.decodedFrame); return fillResult; @@ -966,7 +1139,7 @@ namespace streaming { } av_packet_unref(video_.packet); if (sendResult < 0) { - logging::warn("stream", std::string("avcodec_send_packet failed for H.264: ") + describe_ffmpeg_error(sendResult)); + logging::warn("stream", std::string("avcodec_send_packet failed for ") + video_.codecName + ": " + describe_ffmpeg_error(sendResult)); return DR_NEED_IDR; } diff --git a/src/streaming/ffmpeg_stream_backend.h b/src/streaming/ffmpeg_stream_backend.h index 554f3c2..e82fae6 100644 --- a/src/streaming/ffmpeg_stream_backend.h +++ b/src/streaming/ffmpeg_stream_backend.h @@ -15,6 +15,9 @@ #include // local includes +#include "src/streaming/aligned_media_buffer.h" +#include "src/streaming/pvideo_presenter.h" +#include "src/streaming/stats_overlay.h" #include "third-party/moonlight-common-c/src/Limelight.h" struct AVCodecContext; @@ -34,7 +37,7 @@ namespace streaming { * @brief Owns the FFmpeg decode and SDL presentation state for one stream. * * The backend exposes Moonlight-compatible callback tables for video and audio, - * decodes H.264 video and Opus stereo audio with FFmpeg, presents the most + * decodes negotiated video and Opus stereo audio with FFmpeg, presents the most * recent decoded frame through SDL, and queues decoded PCM samples to SDL audio. */ class FfmpegStreamBackend { @@ -72,6 +75,32 @@ namespace streaming { */ void shutdown(); + /** + * @brief Return retained presentation and decoder acceleration diagnostics. + * @return Snapshot to capture on the render thread before session cleanup. + */ + VideoAccelerationReport acceleration_report() const; + + /** + * @brief Read cumulative performance telemetry. + * + * @return Concurrent video counter snapshot. + */ + VideoStatisticsCounters performance_counters() const; + /** + * @brief Describe the active presentation path. + * + * @return GPU, Software or Waiting. + */ + std::string presentation_path() const; + /** + * @brief Replace the shared GPU/software statistics bitmap on the render thread. + * @param width Bitmap width. + * @param height Bitmap height. + * @param grayscale Packed grayscale pixels with an opaque background. + */ + void set_statistics_bitmap(int width, int height, std::vector grayscale); + /** * @brief Render the latest decoded video frame to the supplied renderer. * @@ -83,6 +112,11 @@ namespace streaming { */ bool render_latest_video_frame(SDL_Renderer *renderer, int screenWidth, int screenHeight, bool allowDirectFramebuffer = true); + /** + * @brief Restore CPU framebuffer output before drawing shutdown or summary UI on the render thread. + */ + void stop_video_presentation(); + /** * @brief Report whether at least one decoded video frame is available. * @@ -90,6 +124,13 @@ namespace streaming { */ bool has_decoded_video() const; + /** + * @brief Return the human-readable active video decoder name. + * + * @return Selected video decoder name, or a generic label before setup. + */ + std::string active_video_decoder_name() const; + /** * @brief Report whether a newly decoded video frame has not been presented yet. * @@ -113,7 +154,7 @@ namespace streaming { std::string build_overlay_status_line() const; /** - * @brief Initialize the FFmpeg H.264 decoder for Moonlight video callbacks. + * @brief Initialize the FFmpeg decoder for Moonlight video callbacks. * * @param videoFormat Negotiated Moonlight video format. * @param width Negotiated stream width. @@ -180,6 +221,27 @@ namespace streaming { void decode_and_play_audio_sample(const char *sampleData, int sampleLength); private: + /** + * @brief Present a frame while the public wrapper records submission time. + * @param renderer SDL video renderer. + * @param screenWidth Output width. + * @param screenHeight Output height. + * @param allowDirectFramebuffer Permit Xbox GPU or direct framebuffer presentation. + * @return True after successful presentation. + */ + bool render_video_frame(SDL_Renderer *renderer, int screenWidth, int screenHeight, bool allowDirectFramebuffer); + /** + * @brief Draw the cached panel when the SDL presentation fallback is used. + * @param renderer SDL video renderer. + * @param destination Output video rectangle. + */ + void render_sdl_statistics(SDL_Renderer *renderer, const SDL_Rect &destination); + /** + * @brief Record a retrieved frame before any client discard. + * + * @param unit Completed network frame. + */ + void record_received_unit(PDECODE_UNIT unit); /** * @brief Entry point used by the SDL video decode worker thread. * @@ -254,9 +316,12 @@ namespace streaming { int yPitch = 0; int uPitch = 0; int vPitch = 0; - std::vector yPlane; - std::vector uPlane; - std::vector vPlane; + AlignedMediaBuffer yPlane; ///< Aligned, padded luma data. + AlignedMediaBuffer uPlane; ///< Aligned, padded blue chroma data. + AlignedMediaBuffer vPlane; ///< Aligned, padded red chroma data. + VideoColorMatrix matrix = VideoColorMatrix::bt601; ///< Color matrix preserved across frame publication. + bool fullRange = false; ///< Source uses full-range YUV. + bool interlaced = false; ///< Source contains interlaced fields. }; /** @@ -269,13 +334,14 @@ namespace streaming { AVFrame *decodedFrame = nullptr; AVFrame *convertedFrame = nullptr; AVPacket *packet = nullptr; + const char *codecName = "video"; ///< Human-readable active video decoder name. SDL_Texture *texture = nullptr; SDL_Thread *decoderThread = nullptr; int textureWidth = 0; int textureHeight = 0; std::uint64_t latestFrameVersion = 0; std::uint64_t renderedFrameVersion = 0; - std::vector convertedBuffer; + AlignedMediaBuffer convertedBuffer; ///< Aligned planar conversion storage. std::vector packetBuffer; /** @@ -318,7 +384,7 @@ namespace streaming { AVPacket *packet = nullptr; SDL_AudioDeviceID deviceId = 0; SDL_AudioSpec obtainedSpec {}; - std::vector outputBuffer; + AlignedMediaBuffer outputBuffer; ///< Aligned resampler output. int resampleInputSampleRate = 0; int resampleInputSampleFormat = -1; int resampleInputChannelCount = 0; @@ -328,6 +394,14 @@ namespace streaming { VideoState video_ {}; AudioState audio_ {}; + PvideoPresenter presenter_; ///< Main-thread GPU presentation resources. + VideoOverlay statisticsOverlay_; ///< Shared cached panel for all presentation paths. + VideoStatistics statistics_; ///< Cumulative decode and presentation telemetry. + SDL_Texture *statisticsTexture_ = nullptr; ///< Cached panel texture for SDL's final fallback. + bool statisticsBitmapChanged_ = false; ///< A panel update requires refreshing a paused video frame. + bool statisticsTextureChanged_ = false; ///< SDL texture requires the current bitmap. + VideoAccelerationReport acceleration_; ///< Session diagnostics survive decoder cleanup. + bool overlayFailed_ = false; ///< Avoid repeating a known failing hardware path this session. std::atomic audioPlaybackEnabled_ = false; }; diff --git a/src/streaming/mpeg2_prediction.cpp b/src/streaming/mpeg2_prediction.cpp new file mode 100644 index 0000000..0a760ef --- /dev/null +++ b/src/streaming/mpeg2_prediction.cpp @@ -0,0 +1,60 @@ +/** + * @file src/streaming/mpeg2_prediction.cpp + * @brief Implements an integer reference for the experimental MPEG-2 prediction stage. + */ +#include "src/streaming/mpeg2_prediction.h" + +#include + +namespace { + /** + * @brief Round a signed half-pixel coordinate toward negative infinity. + * + * @param value Half-pixel coordinate. + * + * @return Integer sample position. + */ + std::int64_t floor_half(std::int64_t value) { + return value >= 0 ? value / 2 : (value - 1) / 2; + } +} // namespace + +namespace streaming { + bool predict_mpeg2_luma(const Mpeg2LumaPlane &reference, int xHalfPel, int yHalfPel, int width, int height, std::uint8_t *destination, std::size_t capacity, int pitch) { + if (reference.data == nullptr || reference.width < 1 || reference.height < 1 || reference.pitch < reference.width || reference.size < static_cast(reference.height - 1) * static_cast(reference.pitch) + static_cast(reference.width) || destination == nullptr || width < 1 || height < 1 || width > 128 || height > 128 || pitch < width || capacity < static_cast(height - 1) * static_cast(pitch) + static_cast(width)) { + return false; + } + for (int row = 0; row < height; ++row) { + const auto sy = static_cast(yHalfPel) + row * 2; + const auto iy = floor_half(sy); + const bool halfY = sy != iy * 2; + for (int column = 0; column < width; ++column) { + const auto sx = static_cast(xHalfPel) + column * 2; + const auto ix = floor_half(sx); + const bool halfX = sx != ix * 2; + const auto sample = [&](std::int64_t x, std::int64_t y) { + return static_cast(reference.data[static_cast(std::clamp(y, 0, reference.height - 1)) * reference.pitch + static_cast(std::clamp(x, 0, reference.width - 1))]); + }; + unsigned int value = sample(ix, iy); + if (halfX && halfY) { + value = (value + sample(ix + 1, iy) + sample(ix, iy + 1) + sample(ix + 1, iy + 1) + 2U) / 4U; + } else if (halfX) { + value = (value + sample(ix + 1, iy) + 1U) / 2U; + } else if (halfY) { + value = (value + sample(ix, iy + 1) + 1U) / 2U; + } + destination[static_cast(row) * pitch + column] = static_cast(value); + } + } + return true; + } + + void add_mpeg2_residual(std::uint8_t *prediction, const std::int16_t *residual, std::size_t count) { + if (prediction == nullptr || residual == nullptr) { + return; + } + for (std::size_t index = 0; index < count; ++index) { + prediction[index] = static_cast(std::clamp(static_cast(prediction[index]) + residual[index], 0, 255)); + } + } +} // namespace streaming diff --git a/src/streaming/mpeg2_prediction.h b/src/streaming/mpeg2_prediction.h new file mode 100644 index 0000000..9166e5c --- /dev/null +++ b/src/streaming/mpeg2_prediction.h @@ -0,0 +1,46 @@ +/** + * @file src/streaming/mpeg2_prediction.h + * @brief Supplies a CPU oracle for progressive MPEG-2 luma prediction experiments. + */ +#pragma once + +#include +#include + +namespace streaming { + /** + * @brief Read-only reference luma plane with explicit storage bounds. + */ + struct Mpeg2LumaPlane { + const std::uint8_t *data = nullptr; ///< Reference pixels. + std::size_t size = 0; ///< Accessible byte count. + int width = 0; ///< Reference width. + int height = 0; ///< Reference height. + int pitch = 0; ///< Positive row stride. + }; + + /** + * @brief Reconstruct progressive frame luma predictions with MPEG-2 rounding and edge extension. + * + * @param reference Decoded reference picture. + * @param xHalfPel Horizontal source origin in half-pixel units, including motion. + * @param yHalfPel Vertical source origin in half-pixel units, including motion. + * @param width Prediction region width, up to 128 pixels. + * @param height Prediction region height, up to 128 pixels. + * @param destination Output pixels. + * @param capacity Accessible output bytes. + * @param pitch Output row stride. + * + * @return True when the complete prediction fits the supplied buffers. + */ + bool predict_mpeg2_luma(const Mpeg2LumaPlane &reference, int xHalfPel, int yHalfPel, int width, int height, std::uint8_t *destination, std::size_t capacity, int pitch); + + /** + * @brief Add inverse-transformed residuals with MPEG-2 byte saturation. + * + * @param prediction Packed prediction pixels, modified in place. + * @param residual Signed residual samples for the same pixels. + * @param count Number of accessible samples in both arrays. + */ + void add_mpeg2_residual(std::uint8_t *prediction, const std::int16_t *residual, std::size_t count); +} // namespace streaming diff --git a/src/streaming/pvideo_presenter.cpp b/src/streaming/pvideo_presenter.cpp new file mode 100644 index 0000000..8c927d6 --- /dev/null +++ b/src/streaming/pvideo_presenter.cpp @@ -0,0 +1,311 @@ +/** + * @file src/streaming/pvideo_presenter.cpp + * @brief Submits bounded YUY2 overlay buffers using documented NV2A PVIDEO registers. + */ +#include "src/streaming/pvideo_presenter.h" + +#ifdef NXDK + #include + #include + #include + #include + #include + #include + +namespace { + constexpr std::uint32_t VIDEO_BUFFER = 0x8700; ///< Overlay buffer submission register. + constexpr std::uint32_t VIDEO_STOP = 0x8704; ///< Normal overlay stop register. + constexpr std::array CRTC_REGISTERS {31, 82, 32, 27}; ///< VGA controls changed by pb_init but not restored by pb_kill. + bool overlayQuarantined = false; ///< A synchronization failure disables overlays until process exit. + std::array quarantinedBuffers {}; ///< Keep uncertain DMA references resident instead of freeing memory still in use. + + /** + * @brief Wait for pbkit's vblank counter without an unbounded event wait or a synthetic raster register. + * + * xemu's raster register increments on reads rather than reporting scanlines, + * so a raster-wrap poll cannot reliably synchronize buffer reuse there. + * + * @return True on a new vblank, or false after 50 milliseconds. + */ + bool wait_for_scanout() { + const DWORD started = GetTickCount(); + const DWORD previous = pb_get_vbl_counter(); + while (GetTickCount() - started < 50U) { + Sleep(1); + if (pb_get_vbl_counter() != previous) { + return true; + } + } + return false; + } + + /** + * @brief Bound GPU command completion before releasing the scanout context. + * + * @return True when the command processor is idle, or false after 100 milliseconds. + */ + bool wait_for_gpu_idle() { + const DWORD started = GetTickCount(); + while (pb_busy()) { + if (GetTickCount() - started >= 100U) { + return false; + } + Sleep(1); + } + return true; + } + + constexpr std::array SAVED_REGISTERS { + 0x8140, + 0x8704, + 0x8900, + 0x8908, + 0x8910, + 0x8918, + 0x8920, + 0x8928, + 0x8930, + 0x8938, + 0x8940, + 0x8948, + 0x8950, + 0x8958, + 0x8904, + 0x890c, + 0x8914, + 0x891c, + 0x8924, + 0x892c, + 0x8934, + 0x893c, + 0x8944, + 0x894c, + 0x8954, + 0x895c, + 0x8088, + 0x808c, + }; ///< Interrupt, buffer and fetch state touched by the overlay presenter. + + /** + * @brief Encode an endpoint-aligned 12.20 scaling increment. + * + * @param source Source extent in pixels. + * @param destination Destination extent in pixels. + * + * @return Hardware scaling increment. + */ + std::uint32_t scaling_increment(int source, int destination) { + return static_cast((static_cast(source - 1) << 20U) / static_cast(destination - 1)); + } +} // namespace +#endif + +namespace streaming { + PvideoPresenter::~PvideoPresenter() { + reset(); + } + + bool PvideoPresenter::active() const { + return active_; + } + + bool PvideoPresenter::initialize_scanout_surface(std::string *error) { +#ifdef NXDK + const VIDEO_MODE mode = XVideoGetMode(); + void *framebuffer = XVideoGetFB(); + if (framebuffer == nullptr || mode.bpp != 32 || mode.width <= 0 || mode.height <= 0 || (mode.width * 4 & 63) != 0) { + *error = "GPU scanout requires a 32-bit framebuffer with a 64-byte aligned pitch"; + return false; + } + for (std::size_t index = 0; index < SAVED_REGISTERS.size(); ++index) { + savedRegisters_[index] = VIDEOREG(SAVED_REGISTERS[index]); + } + const auto crtcIndex = VIDEOREG8(0x6013d4); + for (std::size_t index = 0; index < CRTC_REGISTERS.size(); ++index) { + VIDEOREG8(0x6013d4) = CRTC_REGISTERS[index]; + savedCrtcRegisters_[index] = VIDEOREG8(0x6013d5); + } + VIDEOREG8(0x6013d4) = crtcIndex; + savedCrtcConfig_ = VIDEOREG(0x600804); + scanoutStateSaved_ = true; + pb_set_color_format(NV097_SET_SURFACE_FORMAT_COLOR_LE_A8R8G8B8, false); + const int status = pb_init(); + if (status != 0) { + *error = "GPU scanout initialization failed (pbkit " + std::to_string(status) + ")"; + return false; + } + surfaceActive_ = true; + if (pb_back_buffer_pitch() != static_cast(mode.width * 4)) { + *error = "GPU scanout pitch does not match the active display mode"; + return false; + } + // Keep SDL's CPU framebuffer outside the GPU render targets. Rendering + // into it would leave a cached GPU surface over later CPU menu updates. + pb_fill(0, 0, mode.width, mode.height, 0xff000000U); + if (!wait_for_gpu_idle()) { + *error = "Timed out preparing the GPU scanout surface"; + return false; + } + VIDEOREG(0x600800) = static_cast(MmGetPhysicalAddress(pb_back_buffer())); + return true; +#else + *error = "GPU scanout is unavailable on this platform"; + return false; +#endif + } + + bool PvideoPresenter::reset() { + bool synchronized = true; +#ifdef NXDK + if (active_) { + VIDEOREG(VIDEO_STOP) = 1; + VIDEOREG(VIDEO_BUFFER) = 0; + synchronized = wait_for_scanout(); + if (!synchronized) { + overlayQuarantined = true; + for (std::size_t index = 0; index < buffers_.size(); ++index) { + quarantinedBuffers[index] = buffers_[index]; + buffers_[index] = nullptr; + } + } + } + for (auto *&buffer : buffers_) { + if (buffer != nullptr) { + MmFreeContiguousMemory(buffer); + buffer = nullptr; + } + } + if (surfaceActive_) { + if (wait_for_gpu_idle()) { + pb_kill(); + } else { + overlayQuarantined = true; + synchronized = false; + } + surfaceActive_ = false; + } + if (scanoutStateSaved_) { + for (std::size_t index = 0; index < SAVED_REGISTERS.size(); ++index) { + VIDEOREG(SAVED_REGISTERS[index]) = savedRegisters_[index]; + } + const auto crtcIndex = VIDEOREG8(0x6013d4); + for (std::size_t index = 0; index < CRTC_REGISTERS.size(); ++index) { + VIDEOREG8(0x6013d4) = CRTC_REGISTERS[index]; + VIDEOREG8(0x6013d5) = savedCrtcRegisters_[index]; + } + VIDEOREG8(0x6013d4) = crtcIndex; + VIDEOREG(0x600804) = savedCrtcConfig_; + // pb_kill disables the TV encoder; the CPU framebuffer needs output + // explicitly re-enabled even when the video mode has not changed. + XVideoSetVideoEnable(TRUE); + scanoutStateSaved_ = false; + } +#endif + active_ = false; + bufferSize_ = 0; + nextBuffer_ = 0; + return synchronized; + } + + bool PvideoPresenter::present(const Yuv420FrameView &frame, const VideoRectangle &destination, std::string *error, const OverlaySamplingRegion &sampling, VideoOverlay *overlay) { + std::string failure = overlay_frame_error(frame); + const int sourceWidth = sampling.width == 0 ? frame.width : sampling.width; + const int sourceHeight = sampling.height == 0 ? frame.height : sampling.height; + if (failure.empty() && (destination.x < 0 || destination.y < 0 || destination.width < 2 || destination.height < 2 || destination.width > 4095 || destination.height > 4095 || destination.x > 4095 - destination.width || destination.y > 4095 - destination.height)) { + failure = "Output rectangle is outside the GPU overlay limits"; + } + if (failure.empty() && (sourceWidth < 2 || sourceHeight < 2 || sampling.xHalfPel < 0 || sampling.yHalfPel < 0 || static_cast(sampling.xHalfPel) + 2LL * (sourceWidth - 1) + (sampling.xHalfPel & 1) > 2LL * (frame.width - 1) || static_cast(sampling.yHalfPel) + 2LL * (sourceHeight - 1) + (sampling.yHalfPel & 1) > 2LL * (frame.height - 1))) { + failure = "Source sampling region exceeds the GPU input buffer"; + } +#ifdef NXDK + if (failure.empty() && overlayQuarantined) { + failure = "GPU overlay disabled after a scanout timeout"; + } + const int pitch = failure.empty() ? (frame.width * 2 + 63) & ~63 : 0; + // Retain one cache line beyond the visible rows for fetch padding and an + // inclusive DMA limit that also satisfies xemu's exclusive-end check. + const std::size_t required = failure.empty() ? static_cast(pitch) * static_cast(frame.height) + 64U : 0; + if (failure.empty() && (!active_ || required != bufferSize_ || pitch != pitch_)) { + if (!reset()) { + failure = "GPU overlay teardown timed out"; + } else if ((VIDEOREG(VIDEO_BUFFER) & 0x11U) != 0) { + failure = "Another owner is using the GPU overlay"; + } else if (!initialize_scanout_surface(&failure)) { + // The explicit initialization reason is retained below. + } else { + for (auto *&buffer : buffers_) { + buffer = static_cast(MmAllocateContiguousMemoryEx(required, 0, 0x03ffffff, 64, PAGE_READWRITE | PAGE_WRITECOMBINE)); + if (buffer == nullptr) { + failure = "Not enough contiguous memory for GPU presentation"; + break; + } + std::memset(buffer, 0, required); + } + if (failure.empty()) { + active_ = true; + bufferSize_ = required; + pitch_ = pitch; + // xemu reads the descriptor concurrently with MMIO writes. Keep a + // single DMA window for both buffers so a new address paired with + // the previous frame's limit is still within bounds. + const auto highestAddress = std::max(MmGetPhysicalAddress(buffers_[0]), MmGetPhysicalAddress(buffers_[1])); + VIDEOREG(0x8900) = 0; + VIDEOREG(0x8908) = static_cast(highestAddress + bufferSize_ - 1U); + VIDEOREG(0x8140) = 0; + VIDEOREG(0x8100) = 0x11; + VIDEOREG(0x8088) = (VIDEOREG(0x8088) & 0xf43ff43fU) | 0x04000400U; + VIDEOREG(0x808c) = (VIDEOREG(0x808c) & 0xf40ff40fU) | 0x04000400U; + } + } + } + if (failure.empty()) { + // The previous submission may latch at this blank. Wait before writing + // the retired buffer, rather than overwriting it while scanout still uses it. + if (!wait_for_scanout()) { + failure = "Timed out waiting for GPU scanout; using software presentation"; + } else if (!pack_overlay_yuy2(frame, buffers_[nextBuffer_], bufferSize_, pitch_)) { + failure = "Failed to pack the GPU overlay frame"; + } + } + if (failure.empty()) { + if (overlay != nullptr) { + overlay->draw_yuy2(buffers_[nextBuffer_], bufferSize_, pitch_, frame.width, frame.height, destination); + } + __atomic_thread_fence(__ATOMIC_SEQ_CST); + // Rotate memory buffers through overlay bank zero. xemu composes bank + // zero only; alternating hardware overlay banks hides every other frame. + constexpr std::uint32_t offset = 0; + const auto physical = static_cast(MmGetPhysicalAddress(buffers_[nextBuffer_])); + VIDEOREG(0x8910 + offset) = 0x1000; + VIDEOREG(0x8918 + offset) = 0x1000; + VIDEOREG(0x8920 + offset) = physical; + // SIZE_IN uses luma pixels; YUY2's two-pixel packing affects byte pitch only. + VIDEOREG(0x8928 + offset) = static_cast(frame.width) | (static_cast(frame.height) << 16U); + VIDEOREG(0x8930 + offset) = static_cast(sampling.xHalfPel * 8) | (static_cast(sampling.yHalfPel * 4) << 17U); + VIDEOREG(0x8938 + offset) = scaling_increment(sourceWidth, destination.width); + VIDEOREG(0x8940 + offset) = scaling_increment(sourceHeight, destination.height); + VIDEOREG(0x8948 + offset) = static_cast(destination.x) | (static_cast(destination.y) << 16U); + VIDEOREG(0x8950 + offset) = static_cast(destination.width) | (static_cast(destination.height) << 16U); + const std::uint32_t format = static_cast(pitch_) | (1U << 16U) | (frame.matrix == VideoColorMatrix::bt709 ? 1U << 24U : 0U); + VIDEOREG(0x8958 + offset) = format; + if (VIDEOREG(0x8958 + offset) != format || VIDEOREG(0x8920 + offset) != physical) { + failure = "GPU overlay register verification failed"; + } else { + VIDEOREG(VIDEO_STOP) = 0; + VIDEOREG(VIDEO_BUFFER) = 1U; + nextBuffer_ ^= 1U; + return true; + } + } +#else + if (failure.empty()) { + failure = "GPU overlay is unavailable on this platform"; + } +#endif + reset(); + if (error != nullptr) { + *error = failure; + } + return false; + } +} // namespace streaming diff --git a/src/streaming/pvideo_presenter.h b/src/streaming/pvideo_presenter.h new file mode 100644 index 0000000..0deb3d5 --- /dev/null +++ b/src/streaming/pvideo_presenter.h @@ -0,0 +1,91 @@ +/** + * @file src/streaming/pvideo_presenter.h + * @brief Owns the NV2A overlay independently of SDL framebuffer rendering. + */ +#pragma once + +#include "src/streaming/video_acceleration.h" +#include "src/streaming/video_overlay.h" + +namespace streaming { + /** + * @brief Optional source region for isolated half-pixel prediction experiments. + */ + struct OverlaySamplingRegion { + int xHalfPel = 0; ///< Horizontal source origin in half-pixel units. + int yHalfPel = 0; ///< Vertical source origin in half-pixel units. + int width = 0; ///< Source extent; zero selects the whole frame. + int height = 0; ///< Source extent; zero selects the whole frame. + }; + + /** + * @brief Double-buffered, main-thread-only NV2A YUV overlay presenter. + * + * Supports progressive limited-range BT.601/BT.709 YUV420 with a compatible + * 32-bit output mode. Callers select software presentation when present() + * returns false. Successful register submission does not verify the image + * displayed by the emulator or console. + * + * @see https://github.com/abaire/nxdk_pgraph_tests/blob/main/src/pvideo_control.cpp + * @see https://github.com/xemu-project/xemu/blob/master/hw/xbox/nv2a/pgraph/gl/display.c + */ + class PvideoPresenter { + public: + /** + * @brief Construct an inactive overlay owner. + */ + PvideoPresenter() = default; + /** + * @brief Stop overlay DMA before releasing its contiguous buffers. + */ + ~PvideoPresenter(); + PvideoPresenter(const PvideoPresenter &) = delete; + PvideoPresenter &operator=(const PvideoPresenter &) = delete; + + /** + * @brief Pack and submit one frame, or return an explicit fallback reason. + * + * @param frame Source YUV420 data. + * @param destination Letterboxed output rectangle. + * @param error Receives the reason acceleration could not be used. + * @param sampling Optional crop used by the standalone prediction experiment. + * @param overlay Optional cached statistics bitmap, composited before GPU scanout. + * + * @return True after the overlay registers have accepted the new buffer. + */ + bool present(const Yuv420FrameView &frame, const VideoRectangle &destination, std::string *error, const OverlaySamplingRegion &sampling = {}, VideoOverlay *overlay = nullptr); + + /** + * @brief Stop DMA and release buffers only after a bounded scanout synchronization. + * + * @return False when buffers had to be quarantined after a scanout timeout. + */ + bool reset(); + + /** + * @brief Return whether the overlay currently owns presentation resources. + * @return True after successful initialization until reset. + */ + bool active() const; + + private: + /** + * @brief Select a GPU-rendered scanout surface while retaining SDL's separate CPU framebuffer. + * + * @param error Receives an initialization or GPU completion failure. + * @return True after the background clear has completed. + */ + bool initialize_scanout_surface(std::string *error); + + std::array buffers_ {}; ///< Contiguous write-combined YUY2 storage. + std::array savedRegisters_ {}; ///< Overlay state restored on teardown. + std::array savedCrtcRegisters_ {}; ///< Legacy scanout controls changed by pbkit. + std::uint32_t savedCrtcConfig_ = 0; ///< Scanout start-address latching mode before pbkit. + bool scanoutStateSaved_ = false; ///< Legacy scanout and overlay state must be restored, including initialization failures. + std::size_t bufferSize_ = 0; ///< Allocated size of each DMA buffer. + int pitch_ = 0; ///< Packed source stride. + unsigned int nextBuffer_ = 0; ///< Buffer not currently being scanned out. + bool active_ = false; ///< Registers and buffers are owned by this presenter. + bool surfaceActive_ = false; ///< pbkit owns the GPU context and vblank counter for the scanout surface. + }; +} // namespace streaming diff --git a/src/streaming/session.cpp b/src/streaming/session.cpp index 7c8e919..6c94ca8 100644 --- a/src/streaming/session.cpp +++ b/src/streaming/session.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -199,6 +200,9 @@ namespace { SDL_Renderer *renderer = nullptr; TTF_Font *titleFont = nullptr; TTF_Font *bodyFont = nullptr; + TTF_Font *statisticsFont = nullptr; ///< Font used only when rebuilding the live panel. + bool statisticsEnabled = false; ///< Performance Stats controls the live panel and end metrics. + streaming::VideoStatisticsSampler statisticsSampler; ///< One-second telemetry intervals. StreamControllerHandle controller; streaming::FfmpegStreamBackend mediaBackend {}; bool ttfInitialized = false; @@ -692,6 +696,10 @@ namespace { { resources->controller.close(); } + if (resources->statisticsFont != nullptr) { + TTF_CloseFont(resources->statisticsFont); + resources->statisticsFont = nullptr; + } if (resources->bodyFont != nullptr) { TTF_CloseFont(resources->bodyFont); resources->bodyFont = nullptr; @@ -794,6 +802,13 @@ namespace { } resources->controller.reset(open_primary_controller(preferredController)); + if (resources->statisticsEnabled) { + resources->statisticsFont = TTF_OpenFont(fontPath.c_str(), std::max(12, videoMode.height / 45)); + if (resources->statisticsFont == nullptr) { + resources->statisticsEnabled = false; + logging::warn("stream", std::string("Unable to load the performance overlay font: ") + TTF_GetError()); + } + } return true; } @@ -1463,13 +1478,18 @@ namespace { } /** - * @brief Sample stream statistics using caller-owned storage for the RTP video counters. + * @brief Capture transport and decoder statistics using caller-owned RTP video counters. * - * @param context Configuration of the active stream. - * @param connectionState Current connection state. - * @return Stream statistics with a copy of the current RTP video counters. + * @param context Active stream configuration and host connection details. + * @param connectionState Mutable connection state tracked by the session loop. + * @param mediaBackend FFmpeg backend that owns the active decoder. + * @return Snapshot ready for rendering in diagnostics or end-of-stream summary. */ - streaming::StreamStatisticsSnapshot sample_stream_statistics(const StreamStartContext &context, const StreamConnectionState &connectionState) { + streaming::StreamStatisticsSnapshot sample_stream_statistics( + const StreamStartContext &context, + const StreamConnectionState &connectionState, + const streaming::FfmpegStreamBackend &mediaBackend + ) { streaming::StreamStatisticsSnapshot snapshot { context.streamConfiguration.width, context.streamConfiguration.height, @@ -1483,12 +1503,13 @@ namespace { -1, -1, connectionState.poorConnection.load(), + mediaBackend.active_video_decoder_name(), }; uint32_t estimatedRtt = 0; if (uint32_t estimatedRttVariance = 0; LiGetEstimatedRttInfo(&estimatedRtt, &estimatedRttVariance)) { - (void) estimatedRttVariance; snapshot.roundTripTimeMs = static_cast(estimatedRtt); + snapshot.roundTripVarianceMs = static_cast(estimatedRttVariance); } snapshot.videoQueueDepth = LiGetPendingVideoFrames(); @@ -1498,9 +1519,69 @@ namespace { snapshot.videoPacketsReceived = static_cast(videoStats.packetCountVideo); snapshot.videoPacketsRecovered = static_cast(videoStats.packetCountFecRecovered); snapshot.videoPacketsLost = static_cast(videoStats.packetCountFecFailed); + snapshot.presentationPath = mediaBackend.presentation_path(); + snapshot.decoderFeatures = mediaBackend.acceleration_report().cpuFeatures; return snapshot; } + /** + * @brief Sample telemetry and rasterize one shared panel at most once per second. + * @param context Active stream configuration. + * @param connectionState Current connection quality. + * @param resources Renderer, cached font, telemetry sampler and video backend. + * @return True when the video must be refreshed to display a new panel. + */ + bool update_stream_statistics_overlay(const StreamStartContext &context, const StreamConnectionState &connectionState, StreamUiResources *resources) { + if (resources == nullptr || !resources->statisticsEnabled || resources->statisticsFont == nullptr || !resources->mediaBackend.has_decoded_video()) { + return false; + } + const auto now = PltGetMicroseconds(); + if (!resources->statisticsSampler.due(now)) { + return false; + } + auto snapshot = sample_stream_statistics(context, connectionState, resources->mediaBackend); + resources->statisticsSampler.sample(now, resources->mediaBackend.performance_counters()); + resources->statisticsSampler.apply(&snapshot); + const auto lines = streaming::build_live_stats_overlay_lines(snapshot); + int screenWidth = 0; + int screenHeight = 0; + SDL_GetRendererOutputSize(resources->renderer, &screenWidth, &screenHeight); + constexpr int padding = 6; + const int lineHeight = TTF_FontLineSkip(resources->statisticsFont) + 2; + int width = 0; + for (const auto &line : lines) { + int lineWidth = 0; + TTF_SizeUTF8(resources->statisticsFont, line.c_str(), &lineWidth, nullptr); + width = std::max(width, lineWidth); + } + width = std::min(width + padding * 2, screenWidth - streaming::VideoOverlay::MARGIN * 2); + const int height = std::min(static_cast(lines.size()) * lineHeight + padding * 2, screenHeight - streaming::VideoOverlay::MARGIN * 2); + std::unique_ptr panel(SDL_CreateRGBSurfaceWithFormat(0, width, height, 32, SDL_PIXELFORMAT_ARGB8888), SDL_FreeSurface); + if (!panel) { + logging::warn("stream", std::string("Unable to allocate the performance overlay: ") + SDL_GetError()); + resources->statisticsEnabled = false; + return false; + } + SDL_FillRect(panel.get(), nullptr, SDL_MapRGB(panel->format, 16, 16, 16)); + int top = padding; + for (const auto &line : lines) { + std::unique_ptr text(TTF_RenderUTF8_Blended(resources->statisticsFont, line.c_str(), {255, 255, 255, 255}), SDL_FreeSurface); + if (text) { + SDL_Rect position {padding, top, text->w, text->h}; + SDL_BlitSurface(text.get(), nullptr, panel.get(), &position); + } + top += lineHeight; + } + std::vector grayscale(static_cast(width) * height); + for (int row = 0; row < height; ++row) { + for (int column = 0; column < width; ++column) { + grayscale[static_cast(row) * width + column] = static_cast(panel->pixels)[static_cast(row) * panel->pitch + column * 4]; + } + } + resources->mediaBackend.set_statistics_bitmap(width, height, std::move(grayscale)); + return true; + } + void request_idle_video_refresh_if_needed(const streaming::FfmpegStreamBackend *mediaBackend, Uint32 *lastRequestTicks) { if (mediaBackend == nullptr || lastRequestTicks == nullptr || !mediaBackend->has_decoded_video()) { return; @@ -1549,6 +1630,7 @@ namespace { if (resources == nullptr || resources->renderer == nullptr || resources->titleFont == nullptr || resources->bodyFont == nullptr) { return false; } + update_stream_statistics_overlay(context, connectionState, resources); int screenWidth = 0; int screenHeight = 0; @@ -1588,7 +1670,7 @@ namespace { input::STREAM_EXIT_HINT, }; if (!renderedVideo) { - lines.emplace(lines.begin() + 2, std::string("Mode: ") + std::to_string(context.streamConfiguration.width) + "x" + std::to_string(context.streamConfiguration.height) + " @ " + std::to_string(context.streamConfiguration.fps) + " FPS | H.264 | Stereo"); + lines.emplace(lines.begin() + 2, std::string("Mode: ") + std::to_string(context.streamConfiguration.width) + "x" + std::to_string(context.streamConfiguration.height) + " @ " + std::to_string(context.streamConfiguration.fps) + " FPS | " + (mediaBackend != nullptr ? mediaBackend->active_video_decoder_name() : std::string("video")) + " | Stereo"); lines.emplace(lines.begin() + 4, std::string("Launch mode: ") + (context.serverInformation.rtspSessionUrl != nullptr ? "Session URL supplied by host" : "Default RTSP discovery")); lines.emplace(lines.begin() + 5, "Waiting for the first decoded video frame and audio output."); } @@ -1620,16 +1702,19 @@ namespace { int cursorY = 28; int titleHeight = 0; - render_text_line(resources->renderer, resources->titleFont, "Stream Summary", {{ACCENT_RED, ACCENT_GREEN, ACCENT_BLUE, 0xFF}, 28, cursorY, std::max(1, screenWidth - 56), &titleHeight}); + const bool fallbackUsed = !statisticsSnapshot.accelerationSummary.empty() && statisticsSnapshot.accelerationSummary.front() == "SOFTWARE FALLBACK USED"; + const SDL_Color summaryColor = fallbackUsed ? SDL_Color {0xFF, 0xB4, 0x46, 0xFF} : SDL_Color {ACCENT_RED, ACCENT_GREEN, ACCENT_BLUE, 0xFF}; + render_text_line(resources->renderer, resources->titleFont, fallbackUsed ? "SOFTWARE FALLBACK USED" : "Stream Summary", {summaryColor, 28, cursorY, std::max(1, screenWidth - 56), &titleHeight}); cursorY += titleHeight + 8; - std::vector lines = { - finalMessage, - std::string("Host: ") + host.displayName + " (" + context.address + ")", - std::string("App: ") + app.name, - }; - for (const std::string &line : streaming::build_stats_overlay_lines(statisticsSnapshot)) { - lines.push_back(line); + std::vector lines = statisticsSnapshot.accelerationSummary; + lines.push_back(finalMessage); + lines.push_back(std::string("Host: ") + host.displayName + " (" + context.address + ")"); + lines.push_back(std::string("App: ") + app.name); + if (statisticsSnapshot.showPerformanceMetrics) { + for (const std::string &line : streaming::build_stats_overlay_lines(statisticsSnapshot)) { + lines.push_back(line); + } } lines.emplace_back("Press any button to return."); @@ -1694,6 +1779,7 @@ namespace { if (resources == nullptr || resources->renderer == nullptr || resources->titleFont == nullptr) { return; } + resources->mediaBackend.stop_video_presentation(); int screenWidth = 0; SDL_GetRendererOutputSize(resources->renderer, &screenWidth, nullptr); SDL_SetRenderDrawColor(resources->renderer, BACKGROUND_RED, BACKGROUND_GREEN, BACKGROUND_BLUE, 0xFF); @@ -1774,7 +1860,8 @@ namespace { void render_active_stream_frame_if_needed(const ActiveStreamLoopContext &loopContext, Uint32 *lastIdleVideoIdrRequestTick) { const bool hasDecodedVideo = loopContext.resources.mediaBackend.has_decoded_video(); - if (const bool shouldRenderFrame = !hasDecodedVideo || loopContext.resources.mediaBackend.has_unrendered_video_frame(); shouldRenderFrame) { + const bool statisticsChanged = update_stream_statistics_overlay(loopContext.startContext, loopContext.connectionState, &loopContext.resources); + if (const bool shouldRenderFrame = !hasDecodedVideo || statisticsChanged || loopContext.resources.mediaBackend.has_unrendered_video_frame(); shouldRenderFrame) { render_stream_frame( loopContext.host, loopContext.app, @@ -1886,6 +1973,7 @@ namespace streaming { const VIDEO_MODE &activeStreamVideoMode = scopedStreamVideoMode.active_video_mode(); StreamUiResources resources {}; + resources.statisticsEnabled = settings.showPerformanceStats; if (std::string initializationError; !initialize_stream_ui_resources(window, activeStreamVideoMode, preferredController, &resources, &initializationError)) { assign_status_message(statusMessage, initializationError); logging::error("stream", initializationError); @@ -1943,7 +2031,7 @@ namespace streaming { ); logging::info( "stream", - std::string("Requested stream configuration | resolution=") + std::to_string(startContext.streamConfiguration.width) + "x" + std::to_string(startContext.streamConfiguration.height) + " | fps=" + std::to_string(startContext.streamConfiguration.fps) + " | bitrate=" + std::to_string(startContext.streamConfiguration.bitrate) + " kbps | packetSize=" + std::to_string(startContext.streamConfiguration.packetSize) + " | networkProfile=" + describe_streaming_remotely_mode(startContext.streamConfiguration.streamingRemotely) + " | clientRefreshX100=" + std::to_string(startContext.streamConfiguration.clientRefreshRateX100) + std::string("Requested stream configuration | resolution=") + std::to_string(startContext.streamConfiguration.width) + "x" + std::to_string(startContext.streamConfiguration.height) + " | fps=" + std::to_string(startContext.streamConfiguration.fps) + " | bitrate=" + std::to_string(startContext.streamConfiguration.bitrate) + " kbps" + " | packetSize=" + std::to_string(startContext.streamConfiguration.packetSize) + " | networkProfile=" + describe_streaming_remotely_mode(startContext.streamConfiguration.streamingRemotely) + " | clientRefreshX100=" + std::to_string(startContext.streamConfiguration.clientRefreshRateX100) ); resources.mediaBackend.set_audio_playback_enabled(settings.playAudioOnXbox); logging::info("stream", std::string("Xbox audio playback ") + (settings.playAudioOnXbox ? "enabled" : "disabled for lower decode latency")); @@ -1984,16 +2072,32 @@ namespace streaming { logging::info("stream", std::string(launchResult.resumedSession ? "Resumed stream for " : "Launched stream for ") + app.name + " on " + host.displayName); run_active_stream_loop({settings, host, app, startContext, connectionState, resources}); - const streaming::StreamStatisticsSnapshot finalStatistics = sample_stream_statistics(startContext, connectionState); + streaming::StreamStatisticsSnapshot finalStatistics = sample_stream_statistics(startContext, connectionState, resources.mediaBackend); + resources.statisticsSampler.apply(&finalStatistics); + if (finalStatistics.decodeTimeMs >= 0) { + finalStatistics.decoderLatencyMs = static_cast(finalStatistics.decodeTimeMs + 0.5); + } + if (finalStatistics.hostProcessingMs >= 0) { + finalStatistics.hostLatencyMs = static_cast(finalStatistics.hostProcessingMs + 0.5); + } + logging::info("stream", "Stopping stream transport"); LiStopConnection(); + logging::info("stream", "Stopped stream transport"); active_connection_state() = nullptr; + const streaming::VideoAccelerationReport acceleration = resources.mediaBackend.acceleration_report(); + finalStatistics.accelerationSummary = acceleration.summary_lines(); + finalStatistics.showPerformanceMetrics = settings.showPerformanceStats; - const std::string finalMessage = describe_session_end(connectionState, app.name); + std::string finalMessage = describe_session_end(connectionState, app.name); + if (!acceleration.fallbackReason.empty()) { + finalMessage += " | SOFTWARE FALLBACK USED"; + } logging::info("stream", finalMessage); - startup::log_memory_statistics(); - if (settings.showPerformanceStats) { - show_stream_end_statistics(host, app, startContext, finalStatistics, finalMessage, &resources); + for (const auto &line : finalStatistics.accelerationSummary) { + logging::info("stream", line); } + startup::log_memory_statistics(); + show_stream_end_statistics(host, app, startContext, finalStatistics, finalMessage, &resources); if (returnController != nullptr) { *returnController = resources.controller.take_controller_for_shell(); } diff --git a/src/streaming/session.h b/src/streaming/session.h index 0259855..389aaee 100644 --- a/src/streaming/session.h +++ b/src/streaming/session.h @@ -22,7 +22,7 @@ namespace streaming { * @brief Run one Xbox streaming session for the selected host app. * * The session launches or resumes the selected app on the host, starts the - * Moonlight transport runtime, decodes H.264 video and Opus audio with + * Moonlight transport runtime, decodes negotiated video and Opus audio with * FFmpeg, forwards controller input, renders the latest decoded frame with a * lightweight overlay, and returns once the user stops streaming or the host * terminates the session. @@ -32,7 +32,7 @@ namespace streaming { * * @param window Shared SDL window reused from the shell. * @param videoMode Active Xbox video mode. - * @param settings Active shell settings that control stream resolution, bitrate, frame rate, host audio playback, and the optional end-of-stream stats summary. + * @param settings Active shell settings that control stream resolution, bitrate, frame rate, host audio playback, and optional performance metrics in the end-of-stream summary. * @param host Selected paired host. * @param app Selected host app. * @param clientIdentity Paired client identity used for authenticated launch requests. diff --git a/src/streaming/stats_overlay.cpp b/src/streaming/stats_overlay.cpp index c3979ae..641149e 100644 --- a/src/streaming/stats_overlay.cpp +++ b/src/streaming/stats_overlay.cpp @@ -6,6 +6,7 @@ #include "src/streaming/stats_overlay.h" // standard includes +#include #include #include #include @@ -29,16 +30,167 @@ namespace { return stream.str(); } + /** + * @brief Format an available measurement without inventing a zero for missing data. + * + * @param value Measurement, or a negative unavailable sentinel. + * @param suffix Unit suffix. + * @return One-decimal measurement or N/A. + */ + std::string metric(double value, const char *suffix) { + if (!(value >= 0) || value > 1000000000.0) { + return "N/A"; + } + // nxdk's PDCLib does not consume floating-point printf arguments. Using + // %.1f%s there makes %s interpret part of the double as a string pointer. + const auto tenths = static_cast(value * 10.0 + 0.5); + return std::to_string(tenths / 10) + "." + std::to_string(tenths % 10) + suffix; + } + + /** + * @brief Subtract concurrent cumulative counters defensively. + * + * @param value Current count. + * @param previous Previous count. + * @return Nonnegative delta. + */ + std::uint64_t delta(std::uint64_t value, std::uint64_t previous) { + return value >= previous ? value - previous : 0; + } + } // namespace namespace streaming { + std::vector build_live_stats_overlay_lines(const StreamStatisticsSnapshot &s) { + return { + "Video stream: " + std::to_string(s.width) + "x" + std::to_string(s.height) + " @ " + std::to_string(s.fps) + " FPS (" + s.videoCodec + ")", + "FPS: incoming " + metric(s.incomingFps, "") + " | decoded " + metric(s.decodingFps, "") + " | rendered " + metric(s.renderingFps, ""), + "Video bitrate: " + metric(s.videoBitrateMbps, " Mbps") + " | Host processing: " + metric(s.hostProcessingMs, " ms"), + "Network latency: " + (s.roundTripTimeMs < 0 ? std::string("N/A") : std::to_string(s.roundTripTimeMs) + " ms") + " | Variance: " + (s.roundTripVarianceMs < 0 ? std::string("N/A") : std::to_string(s.roundTripVarianceMs) + " ms"), + "Decode: " + metric(s.decodeTimeMs, " ms") + " | Decoder queue: " + metric(s.queueTimeMs, " ms") + " | Present: " + metric(s.presentationTimeMs, " ms"), + "Frames dropped: network " + metric(s.networkDropPercent, "%") + " | client " + metric(s.clientDropPercent, "%"), + "Video queue: " + (s.videoQueueDepth < 0 ? std::string("N/A") : std::to_string(s.videoQueueDepth) + " frames") + " | Audio queue: " + (s.audioQueueDurationMs < 0 ? std::string("N/A") : std::to_string(s.audioQueueDurationMs) + " ms"), + "Decoder: FFmpeg CPU / " + s.decoderFeatures + " | Presentation: " + s.presentationPath, + }; + } + + void VideoStatistics::reset() { + receivedFrames_ = 0; + networkDroppedFrames_ = 0; + receivedBytes_ = 0; + decodedFrames_ = 0; + renderedFrames_ = 0; + clientDroppedFrames_ = 0; + hostSamples_ = 0; + hostTimeTenthsMs_ = 0; + decodeSamples_ = 0; + decodeTimeUs_ = 0; + queueTimeUs_ = 0; + presentationTimeUs_ = 0; + haveFrameNumber_ = false; + lastFrameNumber_ = 0; + } + + void VideoStatistics::received(std::uint32_t number, std::uint64_t bytes, std::uint16_t hostTimeTenthsMs) { + const std::uint32_t distance = number - lastFrameNumber_; + if (haveFrameNumber_ && distance > 1 && distance < 0x80000000U) { + networkDroppedFrames_.fetch_add(distance - 1); + } + if (!haveFrameNumber_ || (distance > 0 && distance < 0x80000000U)) { + lastFrameNumber_ = number; + } + haveFrameNumber_ = true; + receivedFrames_.fetch_add(1); + receivedBytes_.fetch_add(bytes); + if (hostTimeTenthsMs != 0) { + hostTimeTenthsMs_.fetch_add(hostTimeTenthsMs); + hostSamples_.fetch_add(1); + } + } + + void VideoStatistics::decoded() { + decodedFrames_.fetch_add(1); + } + + void VideoStatistics::decode_time(std::uint64_t durationUs, std::uint64_t queueUs) { + decodeTimeUs_.fetch_add(durationUs); + queueTimeUs_.fetch_add(queueUs); + decodeSamples_.fetch_add(1); + } + + void VideoStatistics::presented(std::uint64_t durationUs) { + presentationTimeUs_.fetch_add(durationUs); + renderedFrames_.fetch_add(1); + } + + void VideoStatistics::dropped(std::uint64_t count) { + clientDroppedFrames_.fetch_add(count); + } + + VideoStatisticsCounters VideoStatistics::counters() const { + return {receivedFrames_.load(), networkDroppedFrames_.load(), receivedBytes_.load(), decodedFrames_.load(), renderedFrames_.load(), clientDroppedFrames_.load(), hostSamples_.load(), hostTimeTenthsMs_.load(), decodeSamples_.load(), decodeTimeUs_.load(), queueTimeUs_.load(), presentationTimeUs_.load()}; + } + + bool VideoStatisticsSampler::due(std::uint64_t nowUs) const { + return !initialized_ || nowUs < sampledAtUs_ || nowUs - sampledAtUs_ >= 1000000U; + } + + void VideoStatisticsSampler::sample(std::uint64_t nowUs, const VideoStatisticsCounters &c) { + if (!due(nowUs)) { + return; + } + if (initialized_ && nowUs > sampledAtUs_ && c.receivedFrames >= previous_.receivedFrames && c.decodedFrames >= previous_.decodedFrames && c.renderedFrames >= previous_.renderedFrames) { + const double seconds = static_cast(nowUs - sampledAtUs_) / 1000000.0; + const auto received = delta(c.receivedFrames, previous_.receivedFrames); + const auto decoded = delta(c.decodedFrames, previous_.decodedFrames); + const auto rendered = delta(c.renderedFrames, previous_.renderedFrames); + const auto missing = delta(c.networkDroppedFrames, previous_.networkDroppedFrames); + const auto hostSamples = delta(c.hostSamples, previous_.hostSamples); + const auto decodeSamples = delta(c.decodeSamples, previous_.decodeSamples); + measured_.incomingFps = received / seconds; + measured_.decodingFps = decoded / seconds; + measured_.renderingFps = rendered / seconds; + measured_.videoBitrateMbps = delta(c.receivedBytes, previous_.receivedBytes) * 8.0 / seconds / 1000000.0; + measured_.hostProcessingMs = hostSamples ? delta(c.hostTimeTenthsMs, previous_.hostTimeTenthsMs) / (hostSamples * 10.0) : -1; + measured_.decodeTimeMs = decodeSamples ? delta(c.decodeTimeUs, previous_.decodeTimeUs) / (decodeSamples * 1000.0) : -1; + measured_.queueTimeMs = decodeSamples ? delta(c.queueTimeUs, previous_.queueTimeUs) / (decodeSamples * 1000.0) : -1; + measured_.presentationTimeMs = rendered ? delta(c.presentationTimeUs, previous_.presentationTimeUs) / (rendered * 1000.0) : -1; + measured_.networkDropPercent = received + missing ? missing * 100.0 / (received + missing) : -1; + measured_.clientDropPercent = received ? std::min(100.0, delta(c.clientDroppedFrames, previous_.clientDroppedFrames) * 100.0 / received) : -1; + } else { + measured_ = {}; + } + previous_ = c; + sampledAtUs_ = nowUs; + initialized_ = true; + } + + void VideoStatisticsSampler::apply(StreamStatisticsSnapshot *s) const { + if (s == nullptr) { + return; + } + s->incomingFps = measured_.incomingFps; + s->decodingFps = measured_.decodingFps; + s->renderingFps = measured_.renderingFps; + s->videoBitrateMbps = measured_.videoBitrateMbps; + s->hostProcessingMs = measured_.hostProcessingMs; + s->decodeTimeMs = measured_.decodeTimeMs; + s->queueTimeMs = measured_.queueTimeMs; + s->presentationTimeMs = measured_.presentationTimeMs; + s->networkDropPercent = measured_.networkDropPercent; + s->clientDropPercent = measured_.clientDropPercent; + } + std::vector build_stats_overlay_lines(const StreamStatisticsSnapshot &snapshot) { std::vector lines; { std::ostringstream streamLine; streamLine << "Stream: " << snapshot.width << "x" << snapshot.height << " @ " << snapshot.fps << " FPS"; + if (!snapshot.videoCodec.empty()) { + streamLine << " | Decoder: " << snapshot.videoCodec; + } lines.push_back(streamLine.str()); } diff --git a/src/streaming/stats_overlay.h b/src/streaming/stats_overlay.h index 2ece177..9ed5310 100644 --- a/src/streaming/stats_overlay.h +++ b/src/streaming/stats_overlay.h @@ -5,6 +5,8 @@ #pragma once // standard includes +#include +#include #include #include @@ -12,12 +14,19 @@ namespace streaming { /** * @brief Snapshot of stream telemetry shown in diagnostics or stream summaries. + * + * The requested stream rate is separate from measured delivery, decode and + * presentation rates. Queue timing covers the wait before decoding, rather + * than post-decode frame pacing. Presentation timing measures CPU preparation + * and submission, including a scanout wait when used, rather than physical + * display latency. Negative measurements are unavailable; the live panel + * displays N/A until an interval or optional host metadata is available. */ struct StreamStatisticsSnapshot { int width; ///< Stream width in pixels. int height; ///< Stream height in pixels. - int fps; ///< Current stream frame rate. - int roundTripTimeMs; ///< End-to-end measured round-trip latency in milliseconds. + int fps; ///< Requested stream frame rate, not the measured rendering rate. + int roundTripTimeMs; ///< Estimated network round-trip latency in milliseconds. int hostLatencyMs; ///< Host-side processing latency in milliseconds. int decoderLatencyMs; ///< Video decoder latency in milliseconds. int videoQueueDepth; ///< Number of video frames buffered locally. @@ -26,6 +35,22 @@ namespace streaming { int videoPacketsRecovered; ///< Video packets recovered through FEC or retransmission. int videoPacketsLost; ///< Video packets permanently lost. bool poorConnection; ///< True when the host flags the connection as poor. + std::string videoCodec; ///< Human-readable video codec used for the stream. + std::vector accelerationSummary; ///< Retained acceleration paths and fallback reasons. + bool showPerformanceMetrics = true; ///< Optional detailed telemetry; acceleration diagnostics always remain visible. + double incomingFps = -1; ///< Completed frames delivered by the network per second. + double decodingFps = -1; ///< Decoded pictures published per second. + double renderingFps = -1; ///< New pictures submitted for presentation per second. + int roundTripVarianceMs = -1; ///< Estimated variation of network round-trip latency. + double hostProcessingMs = -1; ///< Mean host processing time; absent or zero metadata is excluded. + double decodeTimeMs = -1; ///< Mean CPU decode and frame publication time. + double queueTimeMs = -1; ///< Mean wait between completed frame assembly and starting decode. + double presentationTimeMs = -1; ///< Mean CPU time preparing and submitting a new picture. + double videoBitrateMbps = -1; ///< Received compressed video payload rate, excluding transport headers and FEC. + double networkDropPercent = -1; ///< Missing frame sequence numbers as a percentage of expected frames. + double clientDropPercent = -1; ///< Queue, decode and superseded-picture discards as a percentage of received frames. + std::string presentationPath; ///< Active GPU, software or waiting presentation path. + std::string decoderFeatures; ///< Enabled FFmpeg CPU optimizations. }; /** @@ -39,4 +64,127 @@ namespace streaming { */ std::vector build_stats_overlay_lines(const StreamStatisticsSnapshot &snapshot); + /** + * @brief Build the live performance panel using measured rates and explicit unavailable values. + * + * @param snapshot Current stream measurements. + * @return Compact rows shared by GPU and software presentation. + */ + std::vector build_live_stats_overlay_lines(const StreamStatisticsSnapshot &snapshot); + + /** + * @brief Cumulative video counters sampled without locking the decoder or renderer. + */ + struct VideoStatisticsCounters { + std::uint64_t receivedFrames = 0; ///< Fully assembled network frames observed by the client. + std::uint64_t networkDroppedFrames = 0; ///< Missing frame sequence numbers. + std::uint64_t receivedBytes = 0; ///< Compressed payload bytes, including locally discarded frames. + std::uint64_t decodedFrames = 0; ///< Published decoded pictures. + std::uint64_t renderedFrames = 0; ///< New pictures submitted for display. + std::uint64_t clientDroppedFrames = 0; ///< Queue, decode and presentation discards. + std::uint64_t hostSamples = 0; ///< Frames with host processing metadata. + std::uint64_t hostTimeTenthsMs = 0; ///< Summed host processing time in tenths of milliseconds. + std::uint64_t decodeSamples = 0; ///< Timed decode operations. + std::uint64_t decodeTimeUs = 0; ///< Summed decode time in microseconds. + std::uint64_t queueTimeUs = 0; ///< Summed pre-decode queue time in microseconds. + std::uint64_t presentationTimeUs = 0; ///< Summed presentation preparation and submission time. + }; + + /** + * @brief Collect cumulative telemetry from one decoder worker and one render thread. + * + * Atomic accesses use sequentially consistent ordering. Reading each counter + * independently still allows a snapshot to span multiple worker updates. + */ + class VideoStatistics { + public: + /** + * @brief Reset counters while both workers are stopped. + */ + void reset(); + /** + * @brief Observe every retrieved frame, before any client-side queue dropping. + * @param number Protocol frame sequence number. + * @param bytes Compressed video payload size. + * @param hostTimeTenthsMs Optional host processing time; zero is unavailable. + */ + void received(std::uint32_t number, std::uint64_t bytes, std::uint16_t hostTimeTenthsMs); + /** + * @brief Count a successfully published decoded picture. + */ + void decoded(); + /** + * @brief Record one decode attempt, including frame publication. + * @param durationUs CPU processing duration. + * @param queueUs Time spent waiting for decoding. + */ + void decode_time(std::uint64_t durationUs, std::uint64_t queueUs); + /** + * @brief Record one newly presented picture. + * + * @param durationUs Preparation and submission duration. + */ + void presented(std::uint64_t durationUs); + /** + * @brief Count frames intentionally discarded locally. + * + * @param count Number of frames. + */ + void dropped(std::uint64_t count); + /** + * @brief Read cumulative counters without blocking either video worker. + * + * @return Approximate concurrent snapshot. + */ + VideoStatisticsCounters counters() const; + + private: + std::atomic receivedFrames_ {0}; ///< Network delivery count. + std::atomic networkDroppedFrames_ {0}; ///< Missing sequence count. + std::atomic receivedBytes_ {0}; ///< Delivered payload bytes. + std::atomic decodedFrames_ {0}; ///< Published picture count. + std::atomic renderedFrames_ {0}; ///< Submitted picture count. + std::atomic clientDroppedFrames_ {0}; ///< Local discard count. + std::atomic hostSamples_ {0}; ///< Valid host measurements. + std::atomic hostTimeTenthsMs_ {0}; ///< Host processing sum. + std::atomic decodeSamples_ {0}; ///< Decode timing sample count. + std::atomic decodeTimeUs_ {0}; ///< Decode duration sum. + std::atomic queueTimeUs_ {0}; ///< Queue duration sum. + std::atomic presentationTimeUs_ {0}; ///< Presentation duration sum. + std::uint32_t lastFrameNumber_ = 0; ///< Last sequence observed by the decoder worker. + bool haveFrameNumber_ = false; ///< Sequence baseline is available. + }; + + /** + * @brief Derive one-second interval metrics independently of the requested stream frame rate. + */ + class VideoStatisticsSampler { + public: + /** + * @brief Check whether a new interval can be sampled. + * + * @param nowUs Monotonic clock. + * @return True when sampling is due. + */ + bool due(std::uint64_t nowUs) const; + /** + * @brief Update interval measurements, handling restarts and clock regressions. + * @param nowUs Monotonic sample time. + * @param counters Current cumulative counters. + */ + void sample(std::uint64_t nowUs, const VideoStatisticsCounters &counters); + /** + * @brief Copy measured fields into a display snapshot. + * + * @param snapshot Receives the latest interval measurements. + */ + void apply(StreamStatisticsSnapshot *snapshot) const; + + private: + VideoStatisticsCounters previous_ {}; ///< Previous cumulative sample. + StreamStatisticsSnapshot measured_ {}; ///< Latest interval measurements. + std::uint64_t sampledAtUs_ = 0; ///< Previous monotonic sample time. + bool initialized_ = false; ///< An interval baseline has been captured. + }; + } // namespace streaming diff --git a/src/streaming/video_acceleration.cpp b/src/streaming/video_acceleration.cpp new file mode 100644 index 0000000..e5b4d5c --- /dev/null +++ b/src/streaming/video_acceleration.cpp @@ -0,0 +1,68 @@ +/** + * @file src/streaming/video_acceleration.cpp + * @brief Implements safe overlay packing and stream acceleration diagnostics. + */ +#include "src/streaming/video_acceleration.h" + +namespace streaming { + std::string overlay_frame_error(const Yuv420FrameView &frame) { + if (frame.width < 2 || frame.width > 2046 || frame.height < 2 || frame.height > 2047 || (frame.width & 1) != 0) { + return "Frame dimensions are outside the GPU overlay limits"; + } + if (frame.fullRange || frame.matrix == VideoColorMatrix::unsupported || frame.interlaced) { + return "Source color range, matrix or interlacing requires software conversion"; + } + for (std::size_t plane = 0; plane < frame.planes.size(); ++plane) { + const int width = plane == 0 ? frame.width : frame.width / 2; + const int height = plane == 0 ? frame.height : (frame.height + 1) / 2; + if (frame.planes[plane] == nullptr || frame.pitches[plane] < width || frame.sizes[plane] < static_cast(height - 1) * static_cast(frame.pitches[plane]) + static_cast(width)) { + return "Source plane storage is invalid"; + } + } + return {}; + } + + bool pack_overlay_yuy2(const Yuv420FrameView &frame, std::uint8_t *destination, std::size_t capacity, int pitch) { + const auto *luma = frame.planes[0]; + const auto *blueChroma = frame.planes[1]; + const auto *redChroma = frame.planes[2]; + if (luma == nullptr || blueChroma == nullptr || redChroma == nullptr) { + return false; + } + if (!overlay_frame_error(frame).empty() || destination == nullptr || pitch < frame.width * 2 || capacity < static_cast(frame.height - 1) * static_cast(pitch) + static_cast(frame.width * 2)) { + return false; + } + for (int row = 0; row < frame.height; ++row) { + const auto *y = luma + static_cast(row) * frame.pitches[0]; + const auto *u = blueChroma + static_cast(row / 2) * frame.pitches[1]; + const auto *v = redChroma + static_cast(row / 2) * frame.pitches[2]; + auto *output = destination + static_cast(row) * pitch; + for (int column = 0; column < frame.width; column += 2) { + output[column * 2] = y[column]; + output[column * 2 + 1] = u[column / 2]; + output[column * 2 + 2] = y[column + 1]; + output[column * 2 + 3] = v[column / 2]; + } + } + return true; + } + + void VideoAccelerationReport::record_fallback(const std::string &reason) { + if (fallbackReason.empty()) { + fallbackReason = reason; + } + } + + std::vector VideoAccelerationReport::summary_lines() const { + std::vector lines { + fallbackReason.empty() ? (gpuFrames == 0 && softwareFrames == 0 ? "No video frames were presented" : "GPU presentation: no software fallback") : "SOFTWARE FALLBACK USED", + "GPU overlay: " + std::to_string(gpuFrames) + " frames | Software: " + std::to_string(softwareFrames) + " frames", + }; + if (!fallbackReason.empty()) { + lines.push_back("Reason: " + fallbackReason); + } + lines.push_back("FFmpeg: " + cpuFeatures + (threaded ? " | Threading enabled" : " | THREADING DISABLED")); + lines.emplace_back("Video decoding: CPU (GPU accelerates presentation)"); + return lines; + } +} // namespace streaming diff --git a/src/streaming/video_acceleration.h b/src/streaming/video_acceleration.h new file mode 100644 index 0000000..29de69e --- /dev/null +++ b/src/streaming/video_acceleration.h @@ -0,0 +1,94 @@ +/** + * @file src/streaming/video_acceleration.h + * @brief Describes video planes, overlay packing, and persistent acceleration diagnostics. + */ +#pragma once + +#include +#include +#include +#include +#include + +namespace streaming { + /** + * @brief Color matrices understood by the hardware presentation path. + */ + enum class VideoColorMatrix { + bt601, ///< Limited-range standard-definition YUV. + bt709, ///< Limited-range high-definition YUV. + unsupported, ///< Requires software conversion. + }; + + /** + * @brief Bounds-checked view of a progressive planar YUV420 frame. + */ + struct Yuv420FrameView { + int width = 0; ///< Luma width in pixels. + int height = 0; ///< Luma height in pixels. + std::array planes {}; ///< Y, U and V storage. + std::array sizes {}; ///< Accessible bytes in each plane. + std::array pitches {}; ///< Positive byte strides. + VideoColorMatrix matrix = VideoColorMatrix::bt601; ///< Source conversion matrix. + bool fullRange = false; ///< Full-range YUV requires software conversion. + bool interlaced = false; ///< Interlaced sources require software presentation. + }; + + /** + * @brief Rectangle in output framebuffer pixels. + */ + struct VideoRectangle { + int x = 0; ///< Left edge. + int y = 0; ///< Top edge. + int width = 0; ///< Pixel width. + int height = 0; ///< Pixel height. + }; + + /** + * @brief Validate source bounds and hardware-supported source properties. + * + * @param frame Source plane view. + * + * @return Empty on success, or an actionable software-fallback reason. + */ + std::string overlay_frame_error(const Yuv420FrameView &frame); + + /** + * @brief Pack YUV420 into Y0 U Y1 V bytes without color conversion or scaling. + * + * Missing planes or invalid source and destination bounds are rejected before + * writing to the output buffer. + * + * @param frame Source planes to validate and pack. + * @param destination Output storage. + * @param capacity Accessible destination bytes. + * @param pitch Destination byte stride. + * + * @return True when all source and destination accesses fit their buffers. + */ + bool pack_overlay_yuy2(const Yuv420FrameView &frame, std::uint8_t *destination, std::size_t capacity, int pitch); + + /** + * @brief Accumulates the paths used over an entire stream, including recovered fallbacks. + */ + struct VideoAccelerationReport { + std::uint64_t gpuFrames = 0; ///< New frames submitted to the GPU overlay. + std::uint64_t softwareFrames = 0; ///< New frames presented through software conversion. + std::string fallbackReason; ///< First fallback reason, retained after later successes. + std::string cpuFeatures; ///< CPU optimizations enabled in the linked FFmpeg build. + bool threaded = false; ///< FFmpeg has internal synchronization, independently of the decoder worker count. + + /** + * @brief Retain the first reason for using software presentation. + * + * @param reason Explanation supplied by the failed accelerated path. + */ + void record_fallback(const std::string &reason); + + /** + * @brief Build conspicuous end-of-stream diagnostics. + * @return Text rows suitable for the stream summary and logs. + */ + std::vector summary_lines() const; + }; +} // namespace streaming diff --git a/src/streaming/video_overlay.cpp b/src/streaming/video_overlay.cpp new file mode 100644 index 0000000..be8aea3 --- /dev/null +++ b/src/streaming/video_overlay.cpp @@ -0,0 +1,136 @@ +/** + * @file src/streaming/video_overlay.cpp + * @brief Converts a cached statistics bitmap for GPU scanout and software framebuffers. + */ +#include "src/streaming/video_overlay.h" + +#include +#include +#include + +namespace { + /** + * @brief Validate destination storage before any panel copy. + * @param pixels Destination pointer. + * @param capacity Accessible bytes. + * @param pitch Byte stride. + * @param width Pixel width. + * @param height Pixel height. + * @param bytes Bytes per pixel. + * @return True when every row fits its storage. + */ + bool valid_storage(std::uint8_t *pixels, std::size_t capacity, int pitch, int width, int height, int bytes) { + return pixels != nullptr && width > 0 && height > 0 && bytes > 0 && pitch > 0 && static_cast(pitch) >= static_cast(width) * bytes && + capacity >= static_cast(height - 1) * pitch + static_cast(width) * bytes; + } +} // namespace + +namespace streaming { + void VideoOverlay::set_bitmap(int width, int height, std::vector pixels) { + if (width <= 0 || height <= 0 || width > 2048 || height > 2048 || pixels.size() != static_cast(width) * height) { + width = height = 0; + pixels.clear(); + } + width_ = width; + height_ = height; + grayscale_ = std::move(pixels); + rgb_.clear(); + rgbBpp_ = 0; + yuy2_.clear(); + frameWidth_ = 0; + } + + int VideoOverlay::width() const { + return width_; + } + + int VideoOverlay::height() const { + return height_; + } + + void VideoOverlay::prepare_rgb(int bpp) { + if (rgbBpp_ == bpp) { + return; + } + const int bytes = bpp == 32 ? 4 : 2; + rgb_.resize(grayscale_.size() * bytes); + for (std::size_t i = 0; i < grayscale_.size(); ++i) { + const std::uint32_t gray = grayscale_[i]; + if (bpp == 32) { + const std::uint32_t color = 0xff000000U | gray * 0x010101U; + std::memcpy(rgb_.data() + i * 4, &color, 4); + } else { + const auto color = static_cast(bpp == 16 ? ((gray >> 3U) << 11U) | ((gray >> 2U) << 5U) | (gray >> 3U) : 0x8000U | ((gray >> 3U) << 10U) | ((gray >> 3U) << 5U) | (gray >> 3U)); + std::memcpy(rgb_.data() + i * 2, &color, 2); + } + } + rgbBpp_ = bpp; + } + + const std::vector &VideoOverlay::argb_pixels() { + prepare_rgb(32); + return rgb_; + } + + bool VideoOverlay::draw_rgb(std::uint8_t *pixels, std::size_t capacity, int pitch, int width, int height, int bpp, const VideoRectangle &destination) { + if (grayscale_.empty()) { + return true; + } + const int bytes = bpp == 32 ? 4 : (bpp == 15 || bpp == 16 ? 2 : 0); + if (!valid_storage(pixels, capacity, pitch, width, height, bytes) || destination.x < 0 || destination.y < 0 || destination.width <= MARGIN || destination.height <= MARGIN || destination.x > width - destination.width || destination.y > height - destination.height) { + return false; + } + prepare_rgb(bpp); + const int copyWidth = std::min(width_, destination.width - MARGIN); + const int copyHeight = std::min(height_, destination.height - MARGIN); + const std::size_t columnBytes = (static_cast(destination.x) + MARGIN) * bytes; + const std::size_t copyBytes = static_cast(copyWidth) * bytes; + for (int row = 0; row < copyHeight; ++row) { + const std::size_t framebufferRow = static_cast(destination.y) + MARGIN + row; + std::memcpy(pixels + framebufferRow * pitch + columnBytes, rgb_.data() + static_cast(row) * width_ * bytes, copyBytes); + } + return true; + } + + bool VideoOverlay::draw_yuy2(std::uint8_t *pixels, std::size_t capacity, int pitch, int frameWidth, int frameHeight, const VideoRectangle &destination) { + if (grayscale_.empty()) { + return true; + } + if (!valid_storage(pixels, capacity, pitch, frameWidth, frameHeight, 2) || (frameWidth & 1) != 0 || frameWidth > 2046 || frameHeight > 2047 || destination.width <= MARGIN || destination.height <= MARGIN) { + return false; + } + if (frameWidth_ != frameWidth || frameHeight_ != frameHeight || outputWidth_ != destination.width || outputHeight_ != destination.height) { + frameWidth_ = frameWidth; + frameHeight_ = frameHeight; + outputWidth_ = destination.width; + outputHeight_ = destination.height; + source_.x = ((MARGIN * frameWidth / destination.width) + 1) & ~1; + source_.y = MARGIN * frameHeight / destination.height; + const int visibleWidth = std::min(width_, destination.width - MARGIN); + const int visibleHeight = std::min(height_, destination.height - MARGIN); + source_.width = std::min(frameWidth - source_.x, ((visibleWidth * frameWidth + destination.width - 1) / destination.width + 1) & ~1); + source_.height = std::min(frameHeight - source_.y, (visibleHeight * frameHeight + destination.height - 1) / destination.height); + if (source_.width <= 0 || source_.height <= 0) { + return true; + } + yuy2_.resize(static_cast(source_.width) * source_.height * 2); + for (int row = 0; row < source_.height; ++row) { + const int bitmapRow = std::min(visibleHeight - 1, row * visibleHeight / source_.height); + for (int column = 0; column < source_.width; ++column) { + const int bitmapColumn = std::min(visibleWidth - 1, column * visibleWidth / source_.width); + const auto gray = grayscale_[bitmapRow * width_ + bitmapColumn]; + const std::size_t offset = (static_cast(row) * source_.width + column) * 2; + yuy2_[offset] = static_cast(16 + (gray * 219 + 127) / 255); + yuy2_[offset + 1] = 128; + } + } + } + if (source_.width <= 0 || source_.height <= 0) { + return true; + } + for (int row = 0; row < source_.height; ++row) { + std::memcpy(pixels + static_cast(source_.y + row) * pitch + source_.x * 2, yuy2_.data() + static_cast(row) * source_.width * 2, source_.width * 2); + } + return true; + } +} // namespace streaming diff --git a/src/streaming/video_overlay.h b/src/streaming/video_overlay.h new file mode 100644 index 0000000..596bc3f --- /dev/null +++ b/src/streaming/video_overlay.h @@ -0,0 +1,88 @@ +/** + * @file src/streaming/video_overlay.h + * @brief Caches one grayscale statistics panel for both YUY2 and RGB presentation. + */ +#pragma once + +#include "src/streaming/video_acceleration.h" + +namespace streaming { + /** + * @brief Opaque grayscale panel anchored inside the top-left of the video rectangle. + */ + class VideoOverlay { + public: + /** + * @brief Replace the panel and invalidate its converted caches. + * @param width Bitmap width. + * @param height Bitmap height. + * @param pixels Packed grayscale bytes; invalid storage clears the panel. + */ + void set_bitmap(int width, int height, std::vector pixels); + /** + * @brief Get the panel width. + * + * @return Width, or zero for a cleared panel. + */ + int width() const; + /** + * @brief Get the panel height. + * + * @return Height, or zero for a cleared panel. + */ + int height() const; + /** + * @brief Get cached opaque ARGB pixels for SDL. + * + * @return Packed pixels. + */ + const std::vector &argb_pixels(); + /** + * @brief Composite the panel into a validated packed YUY2 frame before GPU submission. + * @param pixels Destination frame. + * @param capacity Available bytes. + * @param pitch Row stride in bytes. + * @param frameWidth Frame width. + * @param frameHeight Frame height. + * @param destination Output video rectangle controlling panel scale. + * @return True when the panel was copied or is empty. + */ + bool draw_yuy2(std::uint8_t *pixels, std::size_t capacity, int pitch, int frameWidth, int frameHeight, const VideoRectangle &destination); + /** + * @brief Composite the same panel into the final software framebuffer. + * + * Validate storage before writing and calculate byte offsets in size_t. + * + * @param pixels Destination storage. + * @param capacity Available bytes. + * @param pitch Row stride in bytes. + * @param width Framebuffer width. + * @param height Framebuffer height. + * @param bpp Xbox RGB pixel depth: 15, 16 or 32. + * @param destination Output video rectangle. + * @return True when drawing completed or the panel is empty. + */ + bool draw_rgb(std::uint8_t *pixels, std::size_t capacity, int pitch, int width, int height, int bpp, const VideoRectangle &destination); + + static constexpr int MARGIN = 8; ///< Panel inset in output pixels. + + private: + /** + * @brief Convert the grayscale bitmap only when its pixel depth changes. + * + * @param bpp RGB pixel depth. + */ + void prepare_rgb(int bpp); + int width_ = 0; ///< Packed panel width. + int height_ = 0; ///< Packed panel height. + std::vector grayscale_; ///< Font raster and opaque background. + std::vector rgb_; ///< Cached RGB panel. + int rgbBpp_ = 0; ///< Pixel format of the RGB cache. + std::vector yuy2_; ///< Cached source-resolution panel for scanout scaling. + VideoRectangle source_ {}; ///< Panel bounds in source luma pixels. + int frameWidth_ = 0; ///< Source width used by the YUY2 cache. + int frameHeight_ = 0; ///< Source height used by the YUY2 cache. + int outputWidth_ = 0; ///< Output width used by the YUY2 cache. + int outputHeight_ = 0; ///< Output height used by the YUY2 cache. + }; +} // namespace streaming diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 777f645..d344033 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -43,9 +43,11 @@ endif() file(GLOB_RECURSE TEST_SOURCES CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/tests/*.h ${CMAKE_SOURCE_DIR}/tests/*.cpp) +list(FILTER TEST_SOURCES EXCLUDE REGEX "/tests/gpu/") add_executable(${PROJECT_NAME} ${TEST_SOURCES} + "${CMAKE_SOURCE_DIR}/src/_nxdk_compat/ffmpeg_memory.c" ${MOONLIGHT_HOST_TESTABLE_SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES CXX_STANDARD 17) @@ -93,3 +95,22 @@ endif() gtest_discover_tests(${PROJECT_NAME} DISCOVERY_MODE PRE_TEST PROPERTIES ${TEST_RUNTIME_PROPERTIES}) + +# Compile the production MMIO presenter against deterministic Xbox API shims. +# Keep NXDK and the fake Windows headers isolated from ordinary host tests. +add_executable(test_pvideo_presenter + "${CMAKE_SOURCE_DIR}/tests/gpu/pvideo_presenter_test.cpp" + "${CMAKE_SOURCE_DIR}/src/streaming/pvideo_presenter.cpp" + "${CMAKE_SOURCE_DIR}/src/streaming/video_acceleration.cpp" + "${CMAKE_SOURCE_DIR}/src/streaming/video_overlay.cpp") +target_compile_definitions(test_pvideo_presenter PRIVATE NXDK) +target_include_directories(test_pvideo_presenter PRIVATE + "${CMAKE_SOURCE_DIR}/tests/support/pvideo" + "${CMAKE_SOURCE_DIR}") +target_link_libraries(test_pvideo_presenter PRIVATE gtest_main lizardbyte::test_support) +target_compile_options(test_pvideo_presenter PRIVATE ${TEST_COVERAGE_COMPILE_OPTIONS}) +target_link_options(test_pvideo_presenter PRIVATE ${TEST_COVERAGE_LINK_OPTIONS}) + +gtest_discover_tests(test_pvideo_presenter + DISCOVERY_MODE PRE_TEST + PROPERTIES ${TEST_RUNTIME_PROPERTIES}) diff --git a/tests/gpu/pvideo_presenter_test.cpp b/tests/gpu/pvideo_presenter_test.cpp new file mode 100644 index 0000000..4af2e72 --- /dev/null +++ b/tests/gpu/pvideo_presenter_test.cpp @@ -0,0 +1,323 @@ +/** + * @file tests/gpu/pvideo_presenter_test.cpp + * @brief Exercises the production presenter against deterministic NV2A, pbkit and allocator shims. + */ +// test includes +// test includes +#include + +// standard includes +#include +#include +#include +#include + +// lib includes +#include +#include +#include +#include + +// local includes +#include "src/streaming/pvideo_presenter.h" + +namespace { + /** + * @brief Mutable fake hardware, reset independently for each lifecycle test. + */ + struct Hardware { + std::array registers {}; ///< MMIO storage. + std::array crtc {}; ///< Indexed VGA registers. + std::uint8_t crtcIndex = 0; ///< Active VGA selector. + std::array framebuffer {}; ///< CPU-owned framebuffer. + std::array gpuBuffer {}; ///< Separate GPU-owned scanout. + VIDEO_MODE mode {640, 480, 32, 60}; ///< Active output mode. + DWORD ticks = 0; ///< Deterministic clock. + DWORD vblanks = 0; ///< Deterministic scanout clock. + int initResult = 0; ///< Injected pbkit initialization result. + int allocationsUntilFailure = -1; ///< Negative disables allocation failure injection. + int initializations = 0; ///< Number of GPU context acquisitions. + int shutdowns = 0; ///< Number of GPU context releases. + unsigned int pitch = 640 * 4; ///< GPU buffer pitch. + std::uint32_t originalStart = 0; ///< CPU scanout restored by pb_kill. + bool videoEnabled = true; ///< TV encoder output state. + bool backgroundCleared = false; ///< GPU background was cleared before overlay submission. + std::map allocations; ///< Live DMA allocations and addresses. + std::uint32_t nextPhysical = 0x100000; ///< Next fake physical allocation address. + }; + + Hardware hardware; ///< State shared by the hardware API shims. + + /** + * @brief Presenter fixture with a small valid YUV420 picture and clean hardware state. + */ + class PvideoPresenterTest: public testing::Test { + protected: + /** + * @brief Reset hardware without placing its large register bank on the stack. + */ + void SetUp() override { + hardware.registers.fill(0); + hardware.crtc.fill(0); + hardware.crtcIndex = 0; + hardware.mode = {640, 480, 32, 60}; + hardware.ticks = hardware.vblanks = 0; + hardware.initResult = 0; + hardware.allocationsUntilFailure = -1; + hardware.initializations = hardware.shutdowns = 0; + hardware.pitch = 640 * 4; + hardware.videoEnabled = true; + hardware.backgroundCleared = false; + hardware.nextPhysical = 0x100000; + ASSERT_TRUE(hardware.allocations.empty()); + } + + /** + * @brief Require each test to release all DMA storage. + */ + void TearDown() override { + EXPECT_TRUE(hardware.allocations.empty()); + } + + std::array y {16, 81, 145, 235}; ///< Source luma. + std::array chroma {128}; ///< Neutral source chroma. + streaming::Yuv420FrameView frame {2, 2, {y.data(), chroma.data(), chroma.data()}, {4, 1, 1}, {2, 1, 1}}; ///< Valid source. + streaming::VideoRectangle destination {0, 60, 640, 360}; ///< Letterboxed output. + }; +} // namespace + +std::uint32_t &pvideo_test_register(std::uint32_t address) { + return hardware.registers.at(address / 4); +} + +std::uint8_t &pvideo_test_register8(std::uint32_t address) { + return address == 0x6013d4 ? hardware.crtcIndex : hardware.crtc.at(hardware.crtcIndex); +} + +VIDEO_MODE XVideoGetMode() { + return hardware.mode; +} + +unsigned char *XVideoGetFB() { + return hardware.framebuffer.data(); +} + +void XVideoSetVideoEnable(int enable) { + hardware.videoEnabled = enable != 0; +} + +DWORD GetTickCount() { + return hardware.ticks; +} + +void Sleep(DWORD milliseconds) { + hardware.ticks += milliseconds; + ++hardware.vblanks; +} + +std::uint32_t pb_get_vbl_counter() { + return hardware.vblanks; +} + +int pb_busy() { + return 0; +} + +int pb_init() { + ++hardware.initializations; + hardware.originalStart = VIDEOREG(0x600800); + hardware.crtc[31] = 87; + hardware.crtc[82] += 4; + hardware.crtc[32] = 41; + hardware.crtc[27] = 5; + VIDEOREG(0x8910) = 0x1000; + VIDEOREG(0x600804) = 2; + if (hardware.initResult != 0) { + hardware.videoEnabled = false; + } + return hardware.initResult; +} + +void pb_kill() { + ++hardware.shutdowns; + VIDEOREG(0x600800) = hardware.originalStart; + hardware.videoEnabled = false; +} + +void pb_set_color_format(unsigned int format, bool swizzled) { + EXPECT_EQ(format, NV097_SET_SURFACE_FORMAT_COLOR_LE_A8R8G8B8); + EXPECT_FALSE(swizzled); +} + +void pb_fill(int x, int y, int width, int height, std::uint32_t color) { + EXPECT_EQ(x, 0); + EXPECT_EQ(y, 0); + EXPECT_EQ(width, hardware.mode.width); + EXPECT_EQ(height, hardware.mode.height); + EXPECT_EQ(color, 0xff000000U); + hardware.backgroundCleared = true; +} + +std::uint32_t *pb_back_buffer() { + return hardware.gpuBuffer.data(); +} + +unsigned int pb_back_buffer_pitch() { + return hardware.pitch; +} + +void *MmAllocateContiguousMemoryEx(std::size_t size, unsigned int, unsigned int, unsigned int, unsigned int) { + if (hardware.allocationsUntilFailure == 0) { + return nullptr; + } + if (hardware.allocationsUntilFailure > 0) { + --hardware.allocationsUntilFailure; + } + void *buffer = std::malloc(size); + if (buffer != nullptr) { + hardware.allocations.emplace(buffer, hardware.nextPhysical); + hardware.nextPhysical += 0x10000; + } + return buffer; +} + +void MmFreeContiguousMemory(void *buffer) { + EXPECT_EQ(hardware.allocations.erase(buffer), 1U); + std::free(buffer); +} + +std::uintptr_t MmGetPhysicalAddress(void *buffer) { + return buffer == hardware.gpuBuffer.data() ? 0x80000 : hardware.allocations.at(buffer); +} + +/** + * @brief Both RAM buffers are submitted through bank zero with source dimensions in pixels. + */ +TEST_F(PvideoPresenterTest, AlternatesMemoryWithoutSelectingUnsupportedOverlayBank) { + streaming::PvideoPresenter presenter; + std::string error; + ASSERT_TRUE(presenter.present(frame, destination, &error)) << error; + EXPECT_TRUE(hardware.backgroundCleared); + EXPECT_EQ(VIDEOREG(0x600800), 0x80000U); + EXPECT_EQ(VIDEOREG(0x8928), 2U | (2U << 16U)); + EXPECT_EQ(VIDEOREG(0x8700), 1U); + const auto first = VIDEOREG(0x8920); + const auto firstLimit = VIDEOREG(0x8908); + ASSERT_TRUE(presenter.present(frame, destination, &error)) << error; + EXPECT_EQ(VIDEOREG(0x8700), 1U); + EXPECT_NE(VIDEOREG(0x8920), first); + const auto second = VIDEOREG(0x8920); + const auto bytes = 2U * 64U; + // A compositor can retain either frame's limit while reading either address. + EXPECT_LE(second + bytes, firstLimit); + EXPECT_EQ(VIDEOREG(0x8908), firstLimit); + ASSERT_TRUE(presenter.present(frame, destination, &error)) << error; + EXPECT_EQ(VIDEOREG(0x8920), first); + EXPECT_LE(second + bytes, VIDEOREG(0x8908)); + EXPECT_EQ(VIDEOREG(0x8908), firstLimit); + EXPECT_EQ(VIDEOREG(0x8924), 0U); + EXPECT_GE(VIDEOREG(0x8908) - VIDEOREG(0x8920) + 1U, 2U * 64U + 64U); +} + +/** + * @brief The production GPU path composites the cached panel before submitting either buffer. + */ +TEST_F(PvideoPresenterTest, CompositesStatisticsIntoSubmittedVideoBuffer) { + streaming::PvideoPresenter presenter; + streaming::VideoOverlay overlay; + overlay.set_bitmap(2, 2, {255, 255, 0, 0}); + std::string error; + ASSERT_TRUE(presenter.present(frame, destination, &error, {}, &overlay)) << error; + const auto allocation = std::find_if(hardware.allocations.begin(), hardware.allocations.end(), [](const auto &entry) { + return entry.second == VIDEOREG(0x8920); + }); + ASSERT_NE(allocation, hardware.allocations.end()); + const auto *pixels = static_cast(allocation->first); + EXPECT_EQ(pixels[0], 235); + EXPECT_EQ(pixels[1], 128); + EXPECT_EQ(pixels[2], 235); + ASSERT_TRUE(presenter.present(frame, destination, &error)) << error; + const auto next = std::find_if(hardware.allocations.begin(), hardware.allocations.end(), [](const auto &entry) { + return entry.second == VIDEOREG(0x8920); + }); + ASSERT_NE(next, hardware.allocations.end()); + EXPECT_EQ(static_cast(next->first)[2], 81); +} + +/** + * @brief Repeated sessions restore the CPU scanout, legacy registers and enabled video output. + */ +TEST_F(PvideoPresenterTest, RestoresOutputAndRegistersAcrossRepeatedSessions) { + constexpr std::array crtcRegisters {31, 82, 32, 27}; + const std::array originalCrtc {13, 14, 15, 16}; + for (std::size_t index = 0; index < originalCrtc.size(); ++index) { + hardware.crtc[crtcRegisters[index]] = originalCrtc[index]; + } + VIDEOREG(0x8910) = 0x2000; + VIDEOREG(0x600800) = 0x3eb4000; + VIDEOREG(0x600804) = 3; + streaming::PvideoPresenter presenter; + for (int session = 0; session < 2; ++session) { + std::string error; + ASSERT_TRUE(presenter.present(frame, destination, &error)) << error; + ASSERT_TRUE(presenter.reset()); + EXPECT_FALSE(presenter.active()); + EXPECT_EQ(VIDEOREG(0x8700), 0U); + EXPECT_EQ(VIDEOREG(0x8910), 0x2000U); + EXPECT_EQ(VIDEOREG(0x600800), 0x3eb4000U); + EXPECT_EQ(VIDEOREG(0x600804), 3U); + EXPECT_TRUE(hardware.videoEnabled); + EXPECT_TRUE(hardware.allocations.empty()); + for (std::size_t index = 0; index < originalCrtc.size(); ++index) { + EXPECT_EQ(hardware.crtc[crtcRegisters[index]], originalCrtc[index]); + } + } + EXPECT_EQ(hardware.initializations, 2); + EXPECT_EQ(hardware.shutdowns, 2); +} + +/** + * @brief A partial allocation failure releases GPU ownership and restores visible software output. + */ +TEST_F(PvideoPresenterTest, RestoresSoftwareOutputAfterPartialAllocationFailure) { + hardware.allocationsUntilFailure = 1; + streaming::PvideoPresenter presenter; + std::string error; + EXPECT_FALSE(presenter.present(frame, destination, &error)); + EXPECT_NE(error.find("contiguous memory"), std::string::npos); + EXPECT_TRUE(hardware.videoEnabled); + EXPECT_TRUE(hardware.allocations.empty()); + EXPECT_EQ(hardware.shutdowns, 1); +} + +/** + * @brief pbkit failure may disable output internally, so initialization failures also restore it. + */ +TEST_F(PvideoPresenterTest, RestoresSoftwareOutputAfterGpuInitializationFailure) { + hardware.initResult = -4; + streaming::PvideoPresenter presenter; + std::string error; + EXPECT_FALSE(presenter.present(frame, destination, &error)); + EXPECT_NE(error.find("pbkit -4"), std::string::npos); + EXPECT_TRUE(hardware.videoEnabled); + EXPECT_EQ(hardware.shutdowns, 0); +} + +/** + * @brief Unsupported output format and scanout pitch select software with a clear explanation. + */ +TEST_F(PvideoPresenterTest, RejectsIncompatibleOutputBeforeSubmittingOverlay) { + streaming::PvideoPresenter presenter; + std::string error; + hardware.mode.bpp = 16; + EXPECT_FALSE(presenter.present(frame, destination, &error)); + EXPECT_NE(error.find("32-bit framebuffer"), std::string::npos); + EXPECT_EQ(hardware.initializations, 0); + hardware.mode.bpp = 32; + hardware.pitch += 64; + EXPECT_FALSE(presenter.present(frame, destination, &error)); + EXPECT_NE(error.find("pitch"), std::string::npos); + EXPECT_EQ(VIDEOREG(0x8700), 0U); + EXPECT_TRUE(hardware.videoEnabled); + EXPECT_EQ(hardware.shutdowns, 1); +} diff --git a/tests/support/pvideo/hal/video.h b/tests/support/pvideo/hal/video.h new file mode 100644 index 0000000..b375d48 --- /dev/null +++ b/tests/support/pvideo/hal/video.h @@ -0,0 +1,51 @@ +/** + * @file tests/support/pvideo/hal/video.h + * @brief Declares the fake scanout registers used to exercise the real Xbox presenter on the host. + */ +#pragma once + +#include + +/** + * @brief Active framebuffer dimensions and format. + */ +struct VIDEO_MODE { + int width; ///< Width in pixels. + int height; ///< Height in pixels. + int bpp; ///< Bits per pixel. + int refresh; ///< Refresh rate. +}; + +/** + * @brief Access a fake 32-bit NV2A register. + * @param address Register offset. + * @return Mutable register storage. + */ +std::uint32_t &pvideo_test_register(std::uint32_t address); +/** + * @brief Access an indexed CRTC register or its selector. + * @param address Register offset. + * @return Mutable byte register storage. + */ +std::uint8_t &pvideo_test_register8(std::uint32_t address); +#define VIDEOREG(address) pvideo_test_register(address) ///< Redirect MMIO into the fixture. +#define VIDEOREG8(address) pvideo_test_register8(address) ///< Redirect indexed CRTC MMIO into the fixture. + +/** + * @brief Get the fake active output mode. + * + * @return Framebuffer dimensions. + */ +VIDEO_MODE XVideoGetMode(); +/** + * @brief Get the CPU framebuffer. + * + * @return Mutable framebuffer storage. + */ +unsigned char *XVideoGetFB(); +/** + * @brief Enable or disable output. + * + * @param enable Nonzero enables video. + */ +void XVideoSetVideoEnable(int enable); diff --git a/tests/support/pvideo/pbkit/pbkit.h b/tests/support/pvideo/pbkit/pbkit.h new file mode 100644 index 0000000..bcd74f9 --- /dev/null +++ b/tests/support/pvideo/pbkit/pbkit.h @@ -0,0 +1,61 @@ +/** + * @file tests/support/pvideo/pbkit/pbkit.h + * @brief Declares the pbkit calls used by the presenter lifecycle fixture. + */ +#pragma once + +#include + +inline constexpr unsigned int NV097_SET_SURFACE_FORMAT_COLOR_LE_A8R8G8B8 = 8; ///< 32-bit scanout format. + +/** + * @brief Read simulated vertical blanks. + * + * @return Counter advanced by Sleep. + */ +std::uint32_t pb_get_vbl_counter(); +/** + * @brief Query fake command completion. + * + * @return Zero when idle. + */ +int pb_busy(); +/** + * @brief Initialize fake GPU resources. + * + * @return Configured pbkit status. + */ +int pb_init(); +/** + * @brief Release fake GPU resources, disabling video like nxdk. + */ +void pb_kill(); +/** + * @brief Select a color format. + * + * @param format GPU format. + * @param swizzled Texture layout. + */ +void pb_set_color_format(unsigned int format, bool swizzled); +/** + * @brief Clear the GPU background. + * + * @param x Left. + * @param y Top. + * @param width Width. + * @param height Height. + * @param color Packed color. + */ +void pb_fill(int x, int y, int width, int height, std::uint32_t color); +/** + * @brief Get the separate GPU scanout storage. + * + * @return Back buffer. + */ +std::uint32_t *pb_back_buffer(); +/** + * @brief Get GPU row stride. + * + * @return Byte pitch. + */ +unsigned int pb_back_buffer_pitch(); diff --git a/tests/support/pvideo/windows.h b/tests/support/pvideo/windows.h new file mode 100644 index 0000000..55e4432 --- /dev/null +++ b/tests/support/pvideo/windows.h @@ -0,0 +1,25 @@ +/** + * @file tests/support/pvideo/windows.h + * @brief Supplies deterministic time and allocation flags for presenter lifecycle tests. + */ +#pragma once + +#include + +using DWORD = std::uint32_t; ///< Xbox millisecond tick type. +inline constexpr int TRUE = 1; ///< Enabled output. +inline constexpr int PAGE_READWRITE = 4; ///< Writable allocation flag. +inline constexpr int PAGE_WRITECOMBINE = 0x400; ///< Write-combined allocation flag. + +/** + * @brief Get simulated milliseconds. + * + * @return Fixture clock. + */ +DWORD GetTickCount(); +/** + * @brief Advance time and simulated vblank. + * + * @param milliseconds Simulated sleep duration. + */ +void Sleep(DWORD milliseconds); diff --git a/tests/support/pvideo/xboxkrnl/xboxkrnl.h b/tests/support/pvideo/xboxkrnl/xboxkrnl.h new file mode 100644 index 0000000..1cbb3f5 --- /dev/null +++ b/tests/support/pvideo/xboxkrnl/xboxkrnl.h @@ -0,0 +1,32 @@ +/** + * @file tests/support/pvideo/xboxkrnl/xboxkrnl.h + * @brief Declares tracked contiguous-memory operations for presenter lifecycle tests. + */ +#pragma once + +#include +#include + +/** + * @brief Allocate tracked DMA storage. + * @param size Allocation size. + * @param minimum Minimum physical address. + * @param maximum Maximum physical address. + * @param alignment Requested alignment. + * @param flags Memory attributes. + * @return Allocated storage or null when failure is injected. + */ +void *MmAllocateContiguousMemoryEx(std::size_t size, unsigned int minimum, unsigned int maximum, unsigned int alignment, unsigned int flags); +/** + * @brief Free tracked DMA storage. + * + * @param buffer Allocation to release. + */ +void MmFreeContiguousMemory(void *buffer); +/** + * @brief Get a deterministic physical address. + * + * @param buffer Allocated storage. + * @return Fake DMA address. + */ +std::uintptr_t MmGetPhysicalAddress(void *buffer); diff --git a/tests/unit/app/client_state_test.cpp b/tests/unit/app/client_state_test.cpp index 1e85618..a7acd27 100644 --- a/tests/unit/app/client_state_test.cpp +++ b/tests/unit/app/client_state_test.cpp @@ -1071,7 +1071,7 @@ namespace { EXPECT_EQ(state.shell.statusMessage, "Stream frame rate set to 30 FPS"); } - TEST(ClientStateTest, DisplaySettingsCanToggleXboxAudioAndEndStreamStats) { + TEST(ClientStateTest, DisplaySettingsCanToggleXboxAudioAndPerformanceStats) { app::ClientState state = app::create_initial_state(); ASSERT_TRUE(state.settings.playAudioOnXbox); ASSERT_FALSE(state.settings.showPerformanceStats); @@ -1092,9 +1092,9 @@ namespace { EXPECT_EQ(update.navigation.activatedItemId, "toggle-show-performance-stats"); EXPECT_TRUE(update.persistence.settingsChanged); EXPECT_TRUE(state.settings.showPerformanceStats); - EXPECT_EQ(state.shell.statusMessage, "End stream performance stats enabled"); + EXPECT_EQ(state.shell.statusMessage, "Performance stats enabled"); ASSERT_NE(state.detailMenu.selected_item(), nullptr); - EXPECT_EQ(state.detailMenu.selected_item()->label, "Show End Stream Stats: On"); + EXPECT_EQ(state.detailMenu.selected_item()->label, "Performance Stats: On"); } TEST(ClientStateTest, ConfirmationModalCanBeCancelledWithoutRequestingPersistenceChanges) { diff --git a/tests/unit/streaming/ffmpeg_memory_test.cpp b/tests/unit/streaming/ffmpeg_memory_test.cpp new file mode 100644 index 0000000..a2318eb --- /dev/null +++ b/tests/unit/streaming/ffmpeg_memory_test.cpp @@ -0,0 +1,56 @@ +/** + * @file tests/unit/streaming/ffmpeg_memory_test.cpp + * @brief Checks aligned allocation ownership, preservation and failure semantics. + */ +// test includes +// test includes +#include + +// standard includes +#include +#include +#include +#include + +// local includes +#include "src/_nxdk_compat/ffmpeg_memory.h" + +/** + * @brief Growing and shrinking preserve the relevant bytes and requested alignment. + */ +TEST(FfmpegMemoryTest, PreservesDataAcrossAlignedReallocation) { + auto *memory = static_cast(moonlight_ffmpeg_aligned_malloc(73, 64)); + ASSERT_NE(memory, nullptr); + EXPECT_EQ(reinterpret_cast(memory) % 64, 0U); + std::memset(memory, 0x5a, 73); + auto *grown = static_cast(moonlight_ffmpeg_aligned_realloc(memory, 1000, 64)); + ASSERT_NE(grown, nullptr); + EXPECT_EQ(reinterpret_cast(grown) % 64, 0U); + for (std::size_t index = 0; index < 73; ++index) { + EXPECT_EQ(grown[index], 0x5a); + } + auto *shrunk = static_cast(moonlight_ffmpeg_aligned_realloc(grown, 17, 32)); + ASSERT_NE(shrunk, nullptr); + EXPECT_EQ(reinterpret_cast(shrunk) % 32, 0U); + for (std::size_t index = 0; index < 17; ++index) { + EXPECT_EQ(shrunk[index], 0x5a); + } + moonlight_ffmpeg_aligned_free(shrunk); +} + +/** + * @brief Failed reallocations leave the original allocation usable and owned by the caller. + */ +TEST(FfmpegMemoryTest, FailureDoesNotLoseOriginalAllocation) { + auto *memory = static_cast(moonlight_ffmpeg_aligned_malloc(16, 16)); + ASSERT_NE(memory, nullptr); + memory[0] = 42; + EXPECT_EQ(moonlight_ffmpeg_aligned_realloc(memory, std::numeric_limits::max(), 16), nullptr); + EXPECT_EQ(errno, ENOMEM); + EXPECT_EQ(memory[0], 42); + EXPECT_EQ(moonlight_ffmpeg_aligned_realloc(memory, 32, 3), nullptr); + EXPECT_EQ(errno, EINVAL); + EXPECT_EQ(memory[0], 42); + EXPECT_EQ(moonlight_ffmpeg_aligned_realloc(memory, 0, 16), nullptr); + moonlight_ffmpeg_aligned_free(nullptr); +} diff --git a/tests/unit/streaming/live_stats_test.cpp b/tests/unit/streaming/live_stats_test.cpp new file mode 100644 index 0000000..6295fa1 --- /dev/null +++ b/tests/unit/streaming/live_stats_test.cpp @@ -0,0 +1,134 @@ +/** + * @file tests/unit/streaming/live_stats_test.cpp + * @brief Covers telemetry intervals, network loss accounting and unavailable measurements. + */ +// test includes +// test includes +#include + +// standard includes +#include + +// local includes +#include "src/streaming/stats_overlay.h" + +/** + * @brief Rates derive from elapsed time and counters rather than the requested stream FPS. + */ +TEST(LiveStatsTest, MeasuresActualRatesAndStageTimes) { + streaming::VideoStatisticsSampler sampler; + sampler.sample(0, {}); + streaming::StreamStatisticsSnapshot display {}; + display.fps = 60; + sampler.apply(&display); + EXPECT_LT(display.renderingFps, 0); + EXPECT_FALSE(sampler.due(999999)); + streaming::VideoStatisticsCounters counters {30, 2, 500000, 10, 8, 20, 2, 120, 10, 200000, 30000, 24000}; + sampler.sample(1000000, counters); + sampler.apply(&display); + EXPECT_EQ(display.fps, 60); + EXPECT_DOUBLE_EQ(display.incomingFps, 30); + EXPECT_DOUBLE_EQ(display.decodingFps, 10); + EXPECT_DOUBLE_EQ(display.renderingFps, 8); + EXPECT_DOUBLE_EQ(display.videoBitrateMbps, 4); + EXPECT_DOUBLE_EQ(display.hostProcessingMs, 6); + EXPECT_DOUBLE_EQ(display.decodeTimeMs, 20); + EXPECT_DOUBLE_EQ(display.queueTimeMs, 3); + EXPECT_DOUBLE_EQ(display.presentationTimeMs, 3); + EXPECT_DOUBLE_EQ(display.networkDropPercent, 6.25); + EXPECT_NEAR(display.clientDropPercent, 66.6667, 0.001); + counters.receivedFrames += 10; + sampler.sample(2000000, counters); + sampler.apply(&display); + EXPECT_DOUBLE_EQ(display.incomingFps, 10); + EXPECT_DOUBLE_EQ(display.renderingFps, 0); + EXPECT_LT(display.decodeTimeMs, 0); + EXPECT_LT(display.hostProcessingMs, 0); +} + +/** + * @brief Restarts and backwards clocks cannot produce negative or enormous interval rates. + */ +TEST(LiveStatsTest, RebaselinesAfterRestartAndClockRegression) { + streaming::VideoStatisticsSampler sampler; + streaming::VideoStatisticsCounters counters; + counters.receivedFrames = 50; + sampler.sample(2000000, counters); + sampler.sample(3000000, {}); + streaming::StreamStatisticsSnapshot display {}; + sampler.apply(&display); + EXPECT_LT(display.incomingFps, 0); + sampler.sample(1000000, {}); + sampler.apply(&display); + EXPECT_LT(display.incomingFps, 0); + EXPECT_FALSE(sampler.due(1500000)); +} + +/** + * @brief Sequence gaps are distinct from local discards, including wraparound and out-of-order frames. + */ +TEST(LiveStatsTest, SeparatesNetworkLossAndLocalDiscards) { + streaming::VideoStatistics statistics; + statistics.received(100, 1000, 0); + statistics.received(102, 2000, 65); + statistics.received(101, 500, 0); + statistics.received(103, 500, 75); + statistics.dropped(2); + statistics.decoded(); + statistics.decode_time(1200, 300); + statistics.presented(400); + auto counters = statistics.counters(); + EXPECT_EQ(counters.receivedFrames, 4); + EXPECT_EQ(counters.networkDroppedFrames, 1); + EXPECT_EQ(counters.clientDroppedFrames, 2); + EXPECT_EQ(counters.receivedBytes, 4000); + EXPECT_EQ(counters.hostSamples, 2); + EXPECT_EQ(counters.hostTimeTenthsMs, 140); + EXPECT_EQ(counters.decodeTimeUs, 1200); + EXPECT_EQ(counters.presentationTimeUs, 400); + statistics.reset(); + statistics.received(0xfffffffeU, 1, 0); + statistics.received(0, 1, 0); + EXPECT_EQ(statistics.counters().networkDroppedFrames, 1); + EXPECT_EQ(statistics.counters().hostSamples, 0); +} + +/** + * @brief The live panel identifies actual rates and explicit unavailable data on either presentation path. + */ +TEST(LiveStatsTest, FormatsBothPresentationPathsWithoutFabricatedTimings) { + streaming::StreamStatisticsSnapshot snapshot {}; + snapshot.width = 1280; + snapshot.height = 720; + snapshot.fps = 30; + snapshot.videoCodec = "H.264"; + snapshot.decoderFeatures = "MMX/SSE"; + snapshot.roundTripTimeMs = 4; + snapshot.roundTripVarianceMs = 1; + snapshot.renderingFps = 18; + for (const auto *path : {"GPU", "Software"}) { + snapshot.presentationPath = path; + const auto lines = streaming::build_live_stats_overlay_lines(snapshot); + ASSERT_EQ(lines.size(), 8U); + EXPECT_NE(lines[0].find("1280x720 @ 30 FPS"), std::string::npos); + EXPECT_NE(lines[1].find("rendered 18.0"), std::string::npos); + EXPECT_NE(lines[2].find("Host processing: N/A"), std::string::npos); + EXPECT_NE(lines[3].find("Variance: 1 ms"), std::string::npos); + EXPECT_NE(lines[7].find(std::string("Presentation: ") + path), std::string::npos); + } +} + +/** + * @brief Numeric updates use bounded fixed-point text, including rounding and invalid inputs. + */ +TEST(LiveStatsTest, FormatsNumericUpdatesWithoutFloatingPointPrintf) { + streaming::StreamStatisticsSnapshot snapshot {}; + snapshot.incomingFps = 29.96; + snapshot.decodingFps = 18.24; + snapshot.renderingFps = 0; + snapshot.videoBitrateMbps = std::numeric_limits::infinity(); + snapshot.hostProcessingMs = std::numeric_limits::quiet_NaN(); + const auto lines = streaming::build_live_stats_overlay_lines(snapshot); + EXPECT_EQ(lines[1], "FPS: incoming 30.0 | decoded 18.2 | rendered 0.0"); + EXPECT_EQ(lines[2], "Video bitrate: N/A | Host processing: N/A"); +} diff --git a/tests/unit/streaming/mpeg2_prediction_test.cpp b/tests/unit/streaming/mpeg2_prediction_test.cpp new file mode 100644 index 0000000..5fa8d6d --- /dev/null +++ b/tests/unit/streaming/mpeg2_prediction_test.cpp @@ -0,0 +1,50 @@ +/** + * @file tests/unit/streaming/mpeg2_prediction_test.cpp + * @brief Checks MPEG-2 half-pixel rounding, negative motion and residual saturation. + */ +// test includes +// test includes +#include + +// standard includes +#include + +// local includes +#include "src/streaming/mpeg2_prediction.h" + +/** + * @brief Horizontal, vertical and diagonal prediction use the specified integer rounding. + */ +TEST(Mpeg2PredictionTest, RoundsHalfPixelPredictions) { + const std::array pixels {0, 1, 2, 4}; + const streaming::Mpeg2LumaPlane reference {pixels.data(), pixels.size(), 2, 2, 2}; + std::uint8_t prediction = 0; + ASSERT_TRUE(streaming::predict_mpeg2_luma(reference, 1, 0, 1, 1, &prediction, 1, 1)); + EXPECT_EQ(prediction, 1); + ASSERT_TRUE(streaming::predict_mpeg2_luma(reference, 0, 1, 1, 1, &prediction, 1, 1)); + EXPECT_EQ(prediction, 1); + ASSERT_TRUE(streaming::predict_mpeg2_luma(reference, 1, 1, 1, 1, &prediction, 1, 1)); + EXPECT_EQ(prediction, 2); +} + +/** + * @brief Negative motion extends picture edges instead of wrapping or reading outside storage. + */ +TEST(Mpeg2PredictionTest, ExtendsEdgesForNegativeMotion) { + const std::array pixels {20, 40, 60, 80}; + const streaming::Mpeg2LumaPlane reference {pixels.data(), pixels.size(), 2, 2, 2}; + std::array prediction {}; + ASSERT_TRUE(streaming::predict_mpeg2_luma(reference, -1, -1, 2, 2, prediction.data(), prediction.size(), 2)); + EXPECT_EQ(prediction, (std::array {20, 30, 40, 50})); + EXPECT_FALSE(streaming::predict_mpeg2_luma(reference, 0, 0, 2, 2, prediction.data(), 3, 2)); +} + +/** + * @brief Signed inverse-transformed samples saturate to the valid reconstructed byte range. + */ +TEST(Mpeg2PredictionTest, ClipsResidualAddition) { + std::array prediction {0, 255, 100, 100}; + const std::array residual {-32768, 32767, -70, 90}; + streaming::add_mpeg2_residual(prediction.data(), residual.data(), prediction.size()); + EXPECT_EQ(prediction, (std::array {0, 255, 30, 190})); +} diff --git a/tests/unit/streaming/stats_overlay_test.cpp b/tests/unit/streaming/stats_overlay_test.cpp index 3b469af..db27774 100644 --- a/tests/unit/streaming/stats_overlay_test.cpp +++ b/tests/unit/streaming/stats_overlay_test.cpp @@ -23,12 +23,13 @@ namespace { 12, 3, false, + "H.264", }; const std::vector lines = streaming::build_stats_overlay_lines(snapshot); ASSERT_EQ(lines.size(), 5U); - EXPECT_EQ(lines[0], "Stream: 1280x720 @ 60 FPS"); + EXPECT_EQ(lines[0], "Stream: 1280x720 @ 60 FPS | Decoder: H.264"); EXPECT_EQ(lines[1], "Latency: RTT 18 ms | Host 6 ms | Decode 4 ms"); EXPECT_EQ(lines[2], "Queues: Video 2 frames | Audio 15 ms"); EXPECT_EQ(lines[3], "Video packets: 1024 rx | 12 recovered | 3 lost"); diff --git a/tests/unit/streaming/video_acceleration_test.cpp b/tests/unit/streaming/video_acceleration_test.cpp new file mode 100644 index 0000000..3162cf0 --- /dev/null +++ b/tests/unit/streaming/video_acceleration_test.cpp @@ -0,0 +1,116 @@ +/** + * @file tests/unit/streaming/video_acceleration_test.cpp + * @brief Covers overlay buffer boundaries and persistent fallback diagnostics. + */ +// test includes +#include + +// standard includes +#include + +// local includes +#include "src/streaming/aligned_media_buffer.h" +#include "src/streaming/video_acceleration.h" + +/** + * @brief Packing respects source strides, odd heights, chroma reuse and destination padding. + */ +TEST(VideoAccelerationTest, PacksStridedPlanesWithoutOverwritingPadding) { + const std::array y {10, 20, 99, 99, 30, 40, 99, 99, 50, 60, 99, 99}; + const std::array u {70, 99, 80, 99}; + const std::array v {90, 99, 100, 99}; + const streaming::Yuv420FrameView frame {2, 3, {y.data(), u.data(), v.data()}, {y.size(), u.size(), v.size()}, {4, 2, 2}}; + std::array output; + output.fill(0xcc); + ASSERT_TRUE(streaming::pack_overlay_yuy2(frame, output.data(), output.size(), 6)); + EXPECT_EQ(output, (std::array {10, 70, 20, 90, 0xcc, 0xcc, 30, 70, 40, 90, 0xcc, 0xcc, 50, 80, 60, 100, 0xcc, 0xcc})); +} + +/** + * @brief Truncated planes and unsupported dimensions never touch destination storage. + */ +TEST(VideoAccelerationTest, RejectsInvalidBuffersBeforeWriting) { + const std::array y {16, 32, 48, 64}; + const std::array chroma {128}; + streaming::Yuv420FrameView frame {2, 2, {y.data(), chroma.data(), chroma.data()}, {4, 1, 1}, {2, 1, 1}}; + std::array output; + output.fill(0xcc); + EXPECT_FALSE(streaming::pack_overlay_yuy2(frame, output.data(), 7, 4)); + frame.sizes[1] = 0; + EXPECT_FALSE(streaming::pack_overlay_yuy2(frame, output.data(), output.size(), 4)); + frame.width = 3; + EXPECT_FALSE(streaming::pack_overlay_yuy2(frame, output.data(), output.size(), 6)); + for (const auto byte : output) { + EXPECT_EQ(byte, 0xcc); + } +} + +/** + * @brief Each absent source plane is rejected without modifying the destination. + */ +TEST(VideoAccelerationTest, RejectsMissingPlanesBeforeWriting) { + const std::array y {16, 32, 48, 64}; + const std::array chroma {128}; + const streaming::Yuv420FrameView frame {2, 2, {y.data(), chroma.data(), chroma.data()}, {4, 1, 1}, {2, 1, 1}}; + std::array output; + output.fill(0xcc); + const auto unchanged = output; + for (std::size_t plane = 0; plane < frame.planes.size(); ++plane) { + auto invalid = frame; + invalid.planes[plane] = nullptr; + EXPECT_EQ(streaming::overlay_frame_error(invalid), "Source plane storage is invalid"); + EXPECT_FALSE(streaming::pack_overlay_yuy2(invalid, output.data(), output.size(), 4)); + EXPECT_EQ(output, unchanged); + } +} + +/** + * @brief Full-range or interlaced sources explicitly select the software path. + */ +TEST(VideoAccelerationTest, ReportsUnsupportedColorAndInterlacing) { + streaming::Yuv420FrameView frame; + frame.width = 640; + frame.height = 480; + frame.fullRange = true; + EXPECT_NE(streaming::overlay_frame_error(frame).find("color range"), std::string::npos); + frame.fullRange = false; + frame.interlaced = true; + EXPECT_NE(streaming::overlay_frame_error(frame).find("interlacing"), std::string::npos); +} + +/** + * @brief Later GPU successes cannot erase a fallback from the end-of-stream summary. + */ +TEST(VideoAccelerationTest, RetainsFirstFallbackAfterAccelerationRecovers) { + streaming::VideoAccelerationReport report; + report.record_fallback("Not enough contiguous memory"); + report.softwareFrames = 2; + report.gpuFrames = 5; + report.record_fallback("Later failure"); + const auto lines = report.summary_lines(); + ASSERT_GE(lines.size(), 3U); + EXPECT_EQ(lines[0], "SOFTWARE FALLBACK USED"); + EXPECT_NE(lines[1].find("5 frames"), std::string::npos); + EXPECT_EQ(lines[2], "Reason: Not enough contiguous memory"); +} + +/** + * @brief Published plane buffers retain alignment and zero padding after reuse and moves. + */ +TEST(VideoAccelerationTest, KeepsMediaBuffersAlignedAndPadded) { + streaming::AlignedMediaBuffer buffer; + for (const auto size : {1U, 73U, 4097U, 23U}) { + buffer.resize(size); + EXPECT_EQ(reinterpret_cast(buffer.data()) % 64U, 0U); + EXPECT_EQ(buffer.size(), size); + for (std::size_t index = 0; index < streaming::AlignedMediaBuffer::PADDING; ++index) { + EXPECT_EQ(buffer.data()[size + index], 0); + } + buffer.data()[0] = 42; + } + auto moved = std::move(buffer); + EXPECT_EQ(buffer.size(), 0U); + EXPECT_EQ(buffer.data(), nullptr); + EXPECT_EQ(moved.data()[0], 42); + EXPECT_EQ(reinterpret_cast(moved.data()) % 64U, 0U); +} diff --git a/tests/unit/streaming/video_overlay_test.cpp b/tests/unit/streaming/video_overlay_test.cpp new file mode 100644 index 0000000..ea5ac28 --- /dev/null +++ b/tests/unit/streaming/video_overlay_test.cpp @@ -0,0 +1,115 @@ +/** + * @file tests/unit/streaming/video_overlay_test.cpp + * @brief Verifies equivalent GPU/software panel pixels and destination bounds. + */ +// test includes +#include + +// standard includes +#include +#include +#include + +// local includes +#include "src/streaming/video_overlay.h" + +/** + * @brief YUY2 drawing leaves padding untouched and uses limited-range neutral grayscale. + */ +TEST(VideoOverlayTest, CopiesLimitedRangePanelIntoPackedVideo) { + streaming::VideoOverlay overlay; + overlay.set_bitmap(2, 2, {0, 255, 128, 64}); + std::array output; + output.fill(0xcc); + ASSERT_TRUE(overlay.draw_yuy2(output.data(), output.size(), 40, 16, 16, {0, 0, 16, 16})); + EXPECT_EQ(output[8 * 40 + 8 * 2], 16); + EXPECT_EQ(output[8 * 40 + 8 * 2 + 1], 128); + EXPECT_EQ(output[8 * 40 + 9 * 2], 235); + EXPECT_EQ(output[9 * 40 + 8 * 2], 126); + EXPECT_EQ(output[9 * 40 + 9 * 2], 71); + EXPECT_EQ(output[8 * 40 + 7 * 2], 0xcc); + EXPECT_EQ(output[8 * 40 + 10 * 2], 0xcc); + EXPECT_EQ(output[8 * 40 + 39], 0xcc); +} + +/** + * @brief Software output uses the same grayscale panel in every Xbox framebuffer format. + */ +TEST(VideoOverlayTest, DrawsGrayscaleInAllFramebufferFormats) { + streaming::VideoOverlay overlay; + overlay.set_bitmap(2, 1, {0, 255}); + for (const int bpp : {15, 16, 32}) { + const int bytes = bpp == 32 ? 4 : 2; + std::vector output(20 * 20 * bytes, 0xcc); + ASSERT_TRUE(overlay.draw_rgb(output.data(), output.size(), 20 * bytes, 20, 20, bpp, {0, 0, 20, 20})); + std::uint32_t black = 0; + std::uint32_t white = 0; + std::memcpy(&black, output.data() + (8 * 20 + 8) * bytes, bytes); + std::memcpy(&white, output.data() + (8 * 20 + 9) * bytes, bytes); + EXPECT_EQ(black, bpp == 32 ? 0xff000000U : (bpp == 15 ? 0x8000U : 0U)); + EXPECT_EQ(white, bpp == 32 ? 0xffffffffU : 0xffffU); + EXPECT_EQ(output[0], 0xcc); + } +} + +/** + * @brief Offset panels preserve every pixel and padding byte outside the drawing rectangle. + */ +TEST(VideoOverlayTest, DrawsOffsetPanelWithPaddedFramebufferRows) { + streaming::VideoOverlay overlay; + overlay.set_bitmap(3, 2, {0, 255, 128, 255, 128, 0}); + const std::array bitDepths {15, 16, 32}; + const std::array, 3> colors {{{0x8000U, 0xffffU, 0xc210U}, {0x0000U, 0xffffU, 0x8410U}, {0xff000000U, 0xffffffffU, 0xff808080U}}}; + for (std::size_t format = 0; format < bitDepths.size(); ++format) { + const int bpp = bitDepths[format]; + const std::size_t bytes = bpp == 32 ? 4 : 2; + const std::size_t pitch = 32 * bytes + 7; + std::vector output(pitch * 24, 0xcc); + auto expected = output; + const std::array offsets {12 * pitch + 11 * bytes, 12 * pitch + 12 * bytes, 12 * pitch + 13 * bytes, 13 * pitch + 11 * bytes, 13 * pitch + 12 * bytes, 13 * pitch + 13 * bytes}; + const std::array samples {0, 1, 2, 1, 2, 0}; + for (std::size_t pixel = 0; pixel < offsets.size(); ++pixel) { + std::memcpy(expected.data() + offsets[pixel], &colors[format][samples[pixel]], bytes); + } + ASSERT_TRUE(overlay.draw_rgb(output.data(), output.size(), static_cast(pitch), 32, 24, bpp, {3, 4, 15, 14})); + EXPECT_EQ(output, expected); + } +} + +/** + * @brief Scaling is cached in source coordinates and refreshed when text or geometry changes. + */ +TEST(VideoOverlayTest, RefreshesScaledCacheAndHandlesTinyFrames) { + streaming::VideoOverlay overlay; + overlay.set_bitmap(2, 1, {0, 255}); + std::vector output(64 * 64 * 2, 0xcc); + ASSERT_TRUE(overlay.draw_yuy2(output.data(), output.size(), 128, 64, 64, {0, 0, 32, 32})); + EXPECT_EQ(output[16 * 128 + 16 * 2], 16); + EXPECT_EQ(output[16 * 128 + 18 * 2], 235); + overlay.set_bitmap(2, 1, {255, 0}); + ASSERT_TRUE(overlay.draw_yuy2(output.data(), output.size(), 128, 64, 64, {0, 0, 32, 32})); + EXPECT_EQ(output[16 * 128 + 16 * 2], 235); + EXPECT_EQ(output[16 * 128 + 18 * 2], 16); + std::array tiny {}; + EXPECT_TRUE(overlay.draw_yuy2(tiny.data(), tiny.size(), 4, 2, 2, {0, 0, 9, 9})); + EXPECT_TRUE(overlay.draw_yuy2(tiny.data(), tiny.size(), 4, 2, 2, {0, 0, 9, 9})); +} + +/** + * @brief Invalid storage never changes the destination and malformed bitmaps clear the panel. + */ +TEST(VideoOverlayTest, RejectsTruncatedStorageAndClearsInvalidBitmaps) { + streaming::VideoOverlay overlay; + overlay.set_bitmap(2, 2, {0, 255, 0, 255}); + std::array output; + output.fill(0xcc); + EXPECT_FALSE(overlay.draw_yuy2(output.data(), output.size() - 1, 32, 16, 16, {0, 0, 16, 16})); + EXPECT_FALSE(overlay.draw_rgb(output.data(), output.size(), 32, 16, 16, 24, {0, 0, 16, 16})); + EXPECT_FALSE(overlay.draw_rgb(output.data(), output.size(), 32, 16, 16, 16, {-1, 0, 16, 16})); + for (auto byte : output) { + EXPECT_EQ(byte, 0xcc); + } + overlay.set_bitmap(2, 2, {1}); + EXPECT_EQ(overlay.width(), 0); + EXPECT_TRUE(overlay.draw_rgb(nullptr, 0, 0, 0, 0, 0, {})); +}