From 30c59359ac0f7b7630f8a16d01c418934871d24c Mon Sep 17 00:00:00 2001 From: arlophoenix Date: Mon, 27 Jul 2026 21:54:34 +1200 Subject: [PATCH 1/4] Add Disable Creality Stock Services for K1C 2025 Creality's stock 2025 firmware runs a telemetry agent, alchemistp, that uploads printer configuration and logs to Creality, plus WebRTC and AI daemons that go inert once the Built-in Camera Fix takes /dev/video0. Adds a Customize-menu option to disable them and a matching restore, in the shape of the existing Creality Web Interface pair. The rename uses a "disabled." prefix rather than a ".disabled" suffix: rcK iterates over CS??* on this model, which still matches a suffixed name, so a suffix would be a silent no-op. onyxp, thirteenthp and solusp are gated on the Built-in Camera Fix specifically, not on any camera fix - USB Camera Support filters out /dev/video0, so a USB-only install leaves those daemons serving a working built-in camera. klipper, nexusp, quintusp and vectorp are in an enforced refuse-list carrying both the S and CS name forms. Renames are guarded so a failure cannot abort the helper mid-sequence, neither direction overwrites an existing file, the print-state check fails safe to a confirmation prompt on anything it cannot parse, and the completion message reports what actually changed. --- scripts/disable_creality_services.sh | 399 ++++++++++++++++++ .../menu/K1_2025/customize_menu_K1C_2025.sh | 19 + scripts/menu/K1_2025/info_menu_K1C_2025.sh | 23 + scripts/menu/functions.sh | 4 + scripts/paths.sh | 9 + 5 files changed, 454 insertions(+) create mode 100755 scripts/disable_creality_services.sh diff --git a/scripts/disable_creality_services.sh b/scripts/disable_creality_services.sh new file mode 100755 index 0000000..d0f42bc --- /dev/null +++ b/scripts/disable_creality_services.sh @@ -0,0 +1,399 @@ +#!/bin/sh + +set -e + +# The K1C 2025 starts its stock daemons from /usr/apps/etc/init.d via /etc/init.d/rcK, +# which iterates over "$(ls -r .../CS??*)". That glob still matches a ".disabled" +# SUFFIX, so the suffix rename used for the /usr/bin binaries in +# creality_web_interface.sh would be a silent no-op here - the daemon would simply +# start again at the next boot. Every rename below therefore uses the "disabled." +# PREFIX, matching what moonraker_nginx.sh already does for S56moonraker_service and +# what tools_menu_K1C_2025.sh checks for the Klipper configuration lock. +# +# Services this option must never disable, whatever a later edit is tempted to add: +# klipper_service - Klipper itself. +# nexusp_service - the touchscreen backend on :7125; the GUI holds an open +# connection to it. +# quintusp - HAL for the LCD backlight, chassis LED, camera arbitration +# and the power-loss GPIO. Disabling it breaks power-loss +# recovery, and it can cancel a running print. +# gui_service - vectorp, the touchscreen itself. +# Both the S and CS prefixes are listed because the init script names vary by +# firmware - tools.sh and tools_menu_K1C_2025.sh already probe for both forms. +CREALITY_SERVICES_NEVER_DISABLE="S55klipper_service CS55klipper_service S56nexusp_service CS56nexusp_service S59quintusp CS59quintusp S60gui_service CS60gui_service" + +function disable_creality_services_message(){ + top_line + title 'Disable Creality Stock Services' "${yellow}" + inner_line + hr + echo -e " │ ${cyan}Creality's stock firmware runs background daemons that upload ${white}│" + echo -e " │ ${cyan}configuration and logs to Creality, plus WebRTC daemons that ${white}│" + echo -e " │ ${cyan}go inert once the Built-in Camera Fix takes the camera. ${white}│" + echo -e " │ ${cyan}This disables the non-essential ones. It can be undone. ${white}│" + hr + bottom_line +} + +function restore_creality_services_message(){ + top_line + title 'Restore Creality Stock Services' "${yellow}" + inner_line + hr + echo -e " │ ${cyan}This re-enables the Creality stock services turned off by the ${white}│" + echo -e " │ ${cyan}Disable Creality Stock Services option. ${white}│" + hr + bottom_line +} + +function creality_service_disabled_path() { + echo "$(dirname "$1")/disabled.$(basename "$1")" +} + +# The always-attempted set. mDNS is deliberately excluded: it is opt-in, so +# leaving it enabled is the expected outcome and must not read as "not finished". +function creality_core_services() { + echo "$CREALITY_TELEMETRY_SERVICE $CREALITY_CLOUD_WEBRTC_SERVICE $CREALITY_LOCAL_WEBRTC_SERVICE $CREALITY_AI_SERVICE" +} + +# Every service this option is allowed to touch, in rename order. +function creality_stock_services() { + echo "$(creality_core_services) $CREALITY_MDNS_SERVICE" +} + +function creality_services_pending() { + local svc + for svc in $(creality_stock_services); do + if [ -f "$svc" ]; then + return 0 + fi + done + return 1 +} + +function creality_services_disabled_present() { + local svc + for svc in $(creality_stock_services); do + if [ -f "$(creality_service_disabled_path "$svc")" ]; then + return 0 + fi + done + return 1 +} + +# True only when none of the five names exist in either form - i.e. this firmware +# does not ship the daemons this option knows about. Distinguishes "already done" +# from "nothing here to do", which look identical from creality_services_pending. +function creality_services_absent() { + if creality_services_pending || creality_services_disabled_present; then + return 1 + fi + return 0 +} + +# Only the Built-in Camera Fix takes /dev/video0 - files/services/S50builtin_camera-k1c-2025 +# runs mjpg_streamer against it. USB Camera Support deliberately EXCLUDES the built-in +# device (files/services/S50usb_camera-k1c-2025 filters out $BUILTIN_DEV), so a USB-only +# install leaves /dev/video0 free and onyxp/thirteenthp/solusp still serve the Creality +# app camera. Gating on either fix would silently remove a working feature. +function creality_builtin_camera_fix_installed() { + if [ ! -f "$BUILTIN_CAMERA_FILE" ] && [ ! -f "$BUILTIN_CAMERA_LEGACY_FILE" ]; then + return 1 + fi + return 0 +} + +# 0 = printing or paused, 1 = confirmed idle, 2 = could not be determined. +# Both ports are consulted and any "printing" wins: helper Moonraker moves to 7126 on +# this model (moonraker_nginx.sh), while nexusp answers on 7125 whether or not Moonraker +# is installed. Only an explicitly known idle state counts as idle - anything else falls +# through to 2 so the caller asks the user rather than assuming the printer is free. +# jq is not available on the K1_2025 path, so the state is pulled out with sed/grep, and +# the body is trimmed to the print_stats object first so an unrelated "state" key in +# nexusp's response cannot be mistaken for the print state. +function creality_print_in_progress() { + local port body state seen_idle + seen_idle="" + for port in 7126 7125; do + body="$("$CURL" -s -m 3 "http://127.0.0.1:${port}/printer/objects/query?print_stats" 2>/dev/null)" + case "$body" in + *'"print_stats"'*) + ;; + *) + continue;; + esac + state="$(echo "$body" | sed 's/.*"print_stats"//' | grep -o '"state"[[:space:]]*:[[:space:]]*"[A-Za-z_]*"' | head -n 1 | sed 's/.*"\([A-Za-z_]*\)"$/\1/')" + case "$state" in + printing|paused) + return 0;; + standby|complete|completed|cancelled|canceled|error) + seen_idle="1";; + *) + ;; + esac + done + if [ -n "$seen_idle" ]; then + return 1 + fi + return 2 +} + +# Set by creality_disable_one_service / creality_restore_one_service so the caller can +# report what actually happened instead of asserting success. +CREALITY_SERVICES_CHANGED=0 +CREALITY_SERVICES_FAILED=0 +CREALITY_SERVICES_STILL_RUNNING="" + +function creality_disable_one_service() { + local svc="$1" + local proc="$2" + local label="$3" + local disabled_svc never + + # Enforced, not just documented: adding one of these to a tier above cannot + # disable it by accident. + for never in $CREALITY_SERVICES_NEVER_DISABLE; do + if [ "$(basename "$svc")" = "$never" ]; then + error_msg "$never is required by the printer and will not be disabled!" + CREALITY_SERVICES_FAILED=$((CREALITY_SERVICES_FAILED + 1)) + return + fi + done + + disabled_svc="$(creality_service_disabled_path "$svc")" + if [ ! -f "$svc" ]; then + if [ -f "$disabled_svc" ]; then + echo -e "Info: $label is already disabled, skipping..." + else + echo -e "Info: $label is not present on this firmware, skipping..." + fi + return + fi + if [ -f "$disabled_svc" ]; then + echo -e "${yellow}Warning: both $(basename "$svc") and $(basename "$disabled_svc") exist.${white}" + echo -e "${yellow}The firmware likely recreated it. Leaving it alone to avoid losing the backup.${white}" + CREALITY_SERVICES_FAILED=$((CREALITY_SERVICES_FAILED + 1)) + return + fi + + echo -e "Info: Stopping and disabling $label..." + set +e + "$svc" stop > /dev/null 2>&1 + if [ -n "$proc" ]; then + killall -q "$proc" + fi + set -e + # Guarded: an unguarded mv would abort the whole helper under set -e (helper.sh + # sets it globally and functions.sh run() calls the action as a bare $1), leaving + # the daemon stopped but not renamed with no message and no menu to return to. + if ! mv "$svc" "$disabled_svc" 2>/dev/null; then + error_msg "Could not rename $(basename "$svc") - is $(dirname "$svc") writable?" + set +e + "$svc" start > /dev/null 2>&1 + set -e + CREALITY_SERVICES_FAILED=$((CREALITY_SERVICES_FAILED + 1)) + return + fi + CREALITY_SERVICES_CHANGED=$((CREALITY_SERVICES_CHANGED + 1)) + if [ -n "$proc" ]; then + set +e + pidof "$proc" > /dev/null 2>&1 + if [ "$?" = "0" ]; then + CREALITY_SERVICES_STILL_RUNNING="$CREALITY_SERVICES_STILL_RUNNING $proc" + fi + set -e + fi +} + +function creality_restore_one_service() { + local svc="$1" + local label="$2" + local disabled_svc + + disabled_svc="$(creality_service_disabled_path "$svc")" + if [ ! -f "$disabled_svc" ]; then + echo -e "Info: $label is not disabled, skipping..." + return + fi + if [ -f "$svc" ]; then + echo -e "${yellow}Warning: $(basename "$svc") already exists - the firmware recreated it.${white}" + echo -e "${yellow}Keeping the newer file; $(basename "$disabled_svc") left in place.${white}" + CREALITY_SERVICES_FAILED=$((CREALITY_SERVICES_FAILED + 1)) + return + fi + echo -e "Info: Restoring and starting $label..." + if ! mv "$disabled_svc" "$svc" 2>/dev/null; then + error_msg "Could not restore $(basename "$svc") - is $(dirname "$svc") writable?" + CREALITY_SERVICES_FAILED=$((CREALITY_SERVICES_FAILED + 1)) + return + fi + CREALITY_SERVICES_CHANGED=$((CREALITY_SERVICES_CHANGED + 1)) + set +e + "$svc" start > /dev/null 2>&1 + set -e +} + +# Shared by both flows. 0 = safe to continue, 1 = caller should return. +function creality_confirm_printer_idle() { + local print_state confirm + set +e + creality_print_in_progress + print_state=$? + set -e + if [ "$print_state" -eq 0 ]; then + error_msg "A print is in progress, please wait until it is finished!" + return 1 + fi + if [ "$print_state" -eq 2 ]; then + echo -e " ${yellow}Warning: printer state could not be read on port 7126 or 7125.${white}" + echo + read -p " ${white}Confirm that no print is running (${yellow}y${white}/${yellow}n${white}): ${yellow}" confirm + echo -e "${white}" + case "${confirm}" in + Y|y) + return 0;; + *) + error_msg "Operation canceled!" + return 1;; + esac + fi + return 0 +} + +function disable_creality_services(){ + disable_creality_services_message + echo + echo -e " ${yellow}Warning: these services live on a partition that survives a factory" + echo -e " reset, so a reset will not bring them back. Use Restore Creality Stock" + echo -e " Services to undo this.${white}" + echo + local yn mdns_yn camera_fix + while true; do + disable_msg "Creality Stock Services" yn + case "${yn}" in + Y|y) + echo -e "${white}" + if ! creality_confirm_printer_idle; then + return + fi + + # Every decision is collected BEFORE the first rename, so an abandoned + # prompt cannot leave the printer half-disabled, and the idle check below + # cannot go stale while the user reads a question. + camera_fix="no" + if creality_builtin_camera_fix_installed; then + camera_fix="yes" + else + echo -e "${yellow}Warning: the Built-in Camera Fix is not installed, so onyxp, thirteenthp${white}" + echo -e "${yellow}and solusp are left running - they still serve the Creality app camera,${white}" + echo -e "${yellow}local WebRTC and AI detection while /dev/video0 is free.${white}" + echo -e "${yellow}(USB Camera Support does not take the built-in camera, so it does not${white}" + echo -e "${yellow}count here.)${white}" + echo + fi + mdns_yn="n" + if [ -f "$CREALITY_MDNS_SERVICE" ]; then + echo -e " ${yellow}Disabling mDNS also stops .local resolution and Creality" + echo -e " Print / app discovery on the local network.${white}" + echo + read -p " ${white}Also disable ${green}mDNS advertising ${white}? (${yellow}y${white}/${yellow}n${white}): ${yellow}" mdns_yn + echo -e "${white}" + fi + + # Re-check immediately before mutating: the prompts above may have taken a while. + if ! creality_confirm_printer_idle; then + return + fi + + CREALITY_SERVICES_CHANGED=0 + CREALITY_SERVICES_FAILED=0 + CREALITY_SERVICES_STILL_RUNNING="" + creality_disable_one_service "$CREALITY_TELEMETRY_SERVICE" "alchemistp" "Creality telemetry agent (alchemistp)" + if [ "$camera_fix" = "yes" ]; then + creality_disable_one_service "$CREALITY_CLOUD_WEBRTC_SERVICE" "onyxp" "Creality cloud WebRTC signalling (onyxp)" + creality_disable_one_service "$CREALITY_LOCAL_WEBRTC_SERVICE" "thirteenthp" "Creality local WebRTC media server (thirteenthp)" + creality_disable_one_service "$CREALITY_AI_SERVICE" "solusp" "Creality AI failure detection (solusp)" + fi + case "${mdns_yn}" in + Y|y) + creality_disable_one_service "$CREALITY_MDNS_SERVICE" "mdns" "Creality mDNS advertising (mdns)";; + *) + if [ -f "$CREALITY_MDNS_SERVICE" ]; then + echo -e "Info: Leaving mDNS advertising enabled..." + fi;; + esac + + if [ "$CREALITY_SERVICES_CHANGED" -eq 0 ]; then + error_msg "No Creality stock services were disabled!" + if [ "$CREALITY_SERVICES_FAILED" -gt 0 ]; then + echo -e " ${darkred}$CREALITY_SERVICES_FAILED service(s) could not be disabled - see the messages above.${white}" + else + echo -e " ${darkred}None of the expected service files were found on this firmware.${white}" + fi + echo + return + fi + ok_msg "$CREALITY_SERVICES_CHANGED Creality stock service(s) have been disabled successfully!" + if [ "$CREALITY_SERVICES_FAILED" -gt 0 ]; then + echo -e " ${yellow}$CREALITY_SERVICES_FAILED service(s) were skipped - see the messages above.${white}" + fi + if [ -n "$CREALITY_SERVICES_STILL_RUNNING" ]; then + echo -e " ${yellow}Still running:$CREALITY_SERVICES_STILL_RUNNING - please reboot to stop them.${white}" + else + echo -e " ${cyan}The services were stopped, so no reboot is needed.${white}" + fi + echo -e " ${cyan}The change persists across reboots and factory resets until restored.${white}" + return;; + N|n) + error_msg "Disabling canceled!" + return;; + *) + error_msg "Please select a correct choice!";; + esac + done +} + +function restore_creality_services(){ + restore_creality_services_message + local yn + while true; do + restore_msg "Creality Stock Services" yn + case "${yn}" in + Y|y) + echo -e "${white}" + # Restoring restarts onyxp/thirteenthp/solusp, which compete for /dev/video0 + # with the Built-in Camera Fix, so this path needs the same idle check as disable. + if ! creality_confirm_printer_idle; then + return + fi + if creality_builtin_camera_fix_installed; then + echo -e "${yellow}Warning: the Built-in Camera Fix is installed. Restoring these services${white}" + echo -e "${yellow}puts them back in contention for the camera - restart the camera or${white}" + echo -e "${yellow}reboot if the stream misbehaves afterwards.${white}" + echo + fi + CREALITY_SERVICES_CHANGED=0 + CREALITY_SERVICES_FAILED=0 + creality_restore_one_service "$CREALITY_TELEMETRY_SERVICE" "Creality telemetry agent (alchemistp)" + creality_restore_one_service "$CREALITY_CLOUD_WEBRTC_SERVICE" "Creality cloud WebRTC signalling (onyxp)" + creality_restore_one_service "$CREALITY_LOCAL_WEBRTC_SERVICE" "Creality local WebRTC media server (thirteenthp)" + creality_restore_one_service "$CREALITY_AI_SERVICE" "Creality AI failure detection (solusp)" + creality_restore_one_service "$CREALITY_MDNS_SERVICE" "Creality mDNS advertising (mdns)" + if [ "$CREALITY_SERVICES_CHANGED" -eq 0 ]; then + error_msg "No Creality stock services were restored!" + echo + return + fi + ok_msg "$CREALITY_SERVICES_CHANGED Creality stock service(s) have been restored successfully!" + if [ "$CREALITY_SERVICES_FAILED" -gt 0 ]; then + echo -e " ${yellow}$CREALITY_SERVICES_FAILED service(s) were skipped - see the messages above.${white}" + fi + return;; + N|n) + error_msg "Restoration canceled!" + return;; + *) + error_msg "Please select a correct choice!";; + esac + done +} diff --git a/scripts/menu/K1_2025/customize_menu_K1C_2025.sh b/scripts/menu/K1_2025/customize_menu_K1C_2025.sh index 83e7601..577637a 100755 --- a/scripts/menu/K1_2025/customize_menu_K1C_2025.sh +++ b/scripts/menu/K1_2025/customize_menu_K1C_2025.sh @@ -9,6 +9,9 @@ function customize_menu_ui_k1_2025() { hr menu_option '1' 'Install' 'Creality Dynamic Logos for Fluidd' hr + menu_option '2' 'Disable' 'Creality Stock Services' + menu_option '3' 'Restore' 'Creality Stock Services' + hr inner_line hr bottom_menu_option 'b' 'Back to [Main Menu]' "${yellow}" @@ -33,6 +36,22 @@ function customize_menu_k1_2025() { else run "install_creality_dynamic_logos" "customize_menu_ui_k1_2025" fi;; + 2) + if creality_services_absent; then + error_msg "No Creality stock services were found on this firmware!" + elif ! creality_services_pending; then + error_msg "Creality Stock Services are already disabled!" + else + run "disable_creality_services" "customize_menu_ui_k1_2025" + fi;; + 3) + if creality_services_absent; then + error_msg "No Creality stock services were found on this firmware!" + elif ! creality_services_disabled_present; then + error_msg "Creality Stock Services are not disabled!" + else + run "restore_creality_services" "customize_menu_ui_k1_2025" + fi;; B|b) clear; main_menu; break;; Q|q) diff --git a/scripts/menu/K1_2025/info_menu_K1C_2025.sh b/scripts/menu/K1_2025/info_menu_K1C_2025.sh index aee1326..2955beb 100755 --- a/scripts/menu/K1_2025/info_menu_K1C_2025.sh +++ b/scripts/menu/K1_2025/info_menu_K1C_2025.sh @@ -41,6 +41,28 @@ function check_simplyprint_k1_2025() { fi } +# Tri-state: the camera gate makes a partial disable a normal outcome, so a plain +# tick would report a half-done state as finished. mDNS is opt-in and excluded. +function check_creality_services_k1_2025() { + local svc any_disabled any_enabled + any_disabled="" + any_enabled="" + for svc in $(creality_core_services); do + if [ -f "$(creality_service_disabled_path "$svc")" ]; then + any_disabled="1" + elif [ -f "$svc" ]; then + any_enabled="1" + fi + done + if [ -z "$any_disabled" ]; then + echo -e "${red}✗" + elif [ -n "$any_enabled" ]; then + echo -e "${yellow}~" + else + echo -e "${green}✓" + fi +} + function info_menu_ui_k1_2025() { top_line title '[ INFORMATION MENU ]' "${yellow}" @@ -84,6 +106,7 @@ function info_menu_ui_k1_2025() { hr subtitle '•CUSTOMIZATION:' info_line "$(check_file_k1_2025 "$FLUIDD_LOGO_FILE")" 'Creality Dynamic Logos for Fluidd' + info_line "$(check_creality_services_k1_2025)" 'Creality Stock Services Disabled' hr inner_line hr diff --git a/scripts/menu/functions.sh b/scripts/menu/functions.sh index 556f8ee..3644b49 100755 --- a/scripts/menu/functions.sh +++ b/scripts/menu/functions.sh @@ -109,6 +109,10 @@ function remove_msg() { read -p "${white} Are you sure you want to remove ${green}${1} ${white}? (${yellow}y${white}/${yellow}n${white}): ${yellow}" $2 } +function disable_msg() { + read -p "${white} Are you sure you want to disable ${green}${1} ${white}? (${yellow}y${white}/${yellow}n${white}): ${yellow}" $2 +} + function restore_msg() { read -p "${white} Are you sure you want to restore ${green}${1} ${white}? (${yellow}y${white}/${yellow}n${white}): ${yellow}" $2 } diff --git a/scripts/paths.sh b/scripts/paths.sh index 06060df..3748b57 100755 --- a/scripts/paths.sh +++ b/scripts/paths.sh @@ -201,6 +201,15 @@ function set_paths() { # Creality Web Interface # CREALITY_WEB_FILE="${BIN_FOLDER}/web-server" + + # Creality Stock Services (K1C 2025) # + # INITD_FOLDER is already model-branched above; these files only exist on the + # K1_2025, and the option using them is only reachable from its Customize menu. + CREALITY_TELEMETRY_SERVICE="${INITD_FOLDER}/CS61alchemistp_service" + CREALITY_CLOUD_WEBRTC_SERVICE="${INITD_FOLDER}/CS58onyxp_service" + CREALITY_LOCAL_WEBRTC_SERVICE="${INITD_FOLDER}/CS59thirteenthp" + CREALITY_AI_SERVICE="${INITD_FOLDER}/CS57solusp_service" + CREALITY_MDNS_SERVICE="${INITD_FOLDER}/CS99mdns" # Guppy Screen # GUPPY_SCREEN_FOLDER="${USR_DATA}/guppyscreen" From 82fbf822f14b7b6a1aade981958550ea96ad2cb6 Mon Sep 17 00:00:00 2001 From: arlophoenix Date: Mon, 3 Aug 2026 15:09:22 +1200 Subject: [PATCH 2/4] Add Retire Nexusp Backend for K1C 2025 The 2025 runs two Moonrakers against one Klipper: Creality's forked nexusp on :7125 for the touchscreen and the helper's real one on :7126. Querying the wrong port does not fail, it answers - plausibly and wrongly, which has cost real debugging time for Spoolman and for timelapse, and which every user hits the first time they paste a :7125 command from a Klipper forum. This adds an opt-in option that retires nexusp and puts the real Moonraker on :7125, the port the rest of the Klipper ecosystem assumes. The screen is never patched - vectorp hardcodes http://127.0.0.1:7125 and cannot be patched anyway, so what answers there becomes ours. What is load-bearing is not the daemon but two JSON-RPC methods the screen calls that stock Moonraker lacks: server.files.get_directory_ex and server.history.count. creality_compat.py implements them, with 72 offline tests recording behaviour measured against the real nexusp before it was switched off - measurements nobody can re-derive once it is disabled. The print histories are merged first, in both directions: forward at retirement so the screen does not lose everything printed before the helper was installed, and backward at restore so it does not lose everything printed while nexusp was retired. Also fixes S50nginx's reload path, which ran nginx -s reload with no -c and so silently left the old config live. --- README.md | 67 ++ .../creality-compat/creality_compat.py | 674 +++++++++++++ .../creality-compat/merge_job_history.py | 314 ++++++ .../creality-compat/test_creality_compat.py | 945 ++++++++++++++++++ .../creality-compat/test_merge_job_history.py | 459 +++++++++ files/moonraker/moonraker.conf | 5 + files/services/S50nginx | 6 +- scripts/disable_creality_services.sh | 15 +- .../menu/K1_2025/customize_menu_K1C_2025.sh | 22 + scripts/menu/K1_2025/info_menu_K1C_2025.sh | 29 + scripts/moonraker_nginx.sh | 13 + scripts/paths.sh | 22 +- scripts/retire_nexusp.sh | 695 +++++++++++++ 13 files changed, 3257 insertions(+), 9 deletions(-) create mode 100644 files/moonraker/creality-compat/creality_compat.py create mode 100644 files/moonraker/creality-compat/merge_job_history.py create mode 100644 files/moonraker/creality-compat/test_creality_compat.py create mode 100644 files/moonraker/creality-compat/test_merge_job_history.py create mode 100644 scripts/retire_nexusp.sh diff --git a/README.md b/README.md index 6f3ced1..d390671 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,70 @@ This script intended for use on Creality **K1 Series** and **Ender-3 V3 Series** printers allows to add more features. Additional support for K1 2025 by @C0DEbrained. + +## Retire Nexusp Backend (K1C 2025) + +The K1C 2025 runs **two Moonrakers against one Klipper**: Creality's forked +`nexusp` on `:7125` (the touchscreen's backend) and this script's real Moonraker +on `:7126`. They share `-d /usr/data/printer_data`, so one gcode directory and +one klippy socket, but Creality namespaced the databases. + +That split is not merely redundant. Querying the wrong port does not fail — **it +answers**: + +```sh +curl -s http://:7125/server/spoolman/status +# nexusp -> {"error": {"code": 404, "message": "Method not found"}} +``` + +Read at face value that says Spoolman was never connected on this printer. It is +wrong, and every command pasted from a Klipper forum at `:7125` hits it. + +**Customize menu → Retire Nexusp Backend** turns nexusp off and moves the real +Moonraker to `:7125`, the port the rest of the Klipper ecosystem assumes. It is +opt-in and off by default. The touchscreen is never patched — `vectorp` +hardcodes `http://127.0.0.1:7125`, and what answers there becomes ours. + +The option: + +- merges the two print histories before anything is disabled, so the screen does + not lose everything printed before this script was installed; +- installs a small Moonraker component implementing the two JSON-RPC methods the + screen calls and stock Moonraker does not have + (`server.files.get_directory_ex` and `server.history.count`); +- offers to install Pillow into Moonraker's virtualenv. Pillow is **not** in the + Moonraker this script ships, and Moonraker's own thumbnail parser needs it — + without it a freshly uploaded file has no thumbnail on the screen at all; +- renames the nexusp init script rather than deleting anything. + +**Restore Nexusp Backend** undoes all of it, and merges the prints made while +nexusp was retired back into its database first — otherwise the touchscreen's +history would silently stop at the day you retired it. + +### Caveats + +- **A firmware update can put the nexusp service file back.** `/usr/apps/etc/init.d` + survives a factory reset, but an OTA can recreate `CS56nexusp_service` beside + the disabled copy. It then loses the race for `:7125` to Moonraker and dies + silently at every boot. The Information menu reports this with `~`, and + running Retire Nexusp Backend again repairs it. +- **For about four seconds after a cold boot** the screen polls two methods + (`printer.info`, `printer.objects.list`) that Moonraker only registers once + Klipper connects. It resolves itself and needs no action. +- Two of the four Creality-only RPC methods are deliberately not implemented: + `server.history.debug.job` and `server.debug.status`. Neither is + screen-facing. + +### Running the component's tests + +The component and the history merge ship with their tests beside them. They are +the executable record of what was measured against `nexusp` before it was +switched off — once it is retired those measurements cannot be re-derived +without reviving it. They need only Python and pytest, no printer and no +Moonraker: + +```sh +python3 -m pytest -q files/moonraker/creality-compat +``` + +This repository has no CI, so nothing runs them automatically. diff --git a/files/moonraker/creality-compat/creality_compat.py b/files/moonraker/creality-compat/creality_compat.py new file mode 100644 index 0000000..4c75643 --- /dev/null +++ b/files/moonraker/creality-compat/creality_compat.py @@ -0,0 +1,674 @@ +# creality_compat.py — the two JSON-RPC methods Creality's touchscreen needs and +# upstream Moonraker does not have. A Moonraker component; runs on the K1C 2025 +# only, at components/creality_compat.py, enabled by `[creality_compat]` in +# moonraker.conf. +# +# WHY THIS EXISTS +# --------------- +# The 2025 ships two Moonrakers against one Klipper: Creality's `nexusp` on +# :7125 for the touchscreen, and the helper script's real one on :7126 for +# everything else. That split is an active trap — a query to the "wrong" port +# ANSWERS, plausibly and incorrectly — so the "Retire Nexusp Backend" option +# switches nexusp off and moves real Moonraker onto :7125. The screen +# (`vectorp`) reconnects on its own and works, with two exceptions, both of +# which are calls to methods only Creality's fork had: +# +# server.files.get_directory_ex the file browser's paging/sort/search +# server.history.count called once the moment the screen connects +# +# Without them the browser renders its first page (from stock `server.files.list`) +# and then cannot scroll, filter or search. This file closes that gap. It is a +# COMPATIBILITY SHIM, not a feature: every behaviour below was measured against +# a real nexusp before it was switched off, on one K1C 2025 running +# V1.0.0.22.20250711S. The measurements cannot be re-derived once a user has +# retired nexusp, so test_creality_compat.py beside this file is their only +# executable record. +# +# THE RULES, ALL MEASURED, NONE GUESSED +# ------------------------------------- +# `path`, `start`, `limit` and `order` are ALL required. nexusp rejects a call +# missing `path` with "No data for argument: path" and one missing any of the +# other three with a bare "Invalid parameter" that names nothing — which is why +# the parameter names had to come out of the binary's strings in the first place. +# Being more lenient than the reference was considered and rejected: the only +# client is closed-source, so the safe shim is the one that cannot behave +# differently from what that client has always been handed. +# +# What is listed, in a directory: +# - subdirectories, ALWAYS, and always sorted ahead of every file +# - files whose name ends in `.gcode` — measured: an empty `.gcode` IS listed +# (so this is an extension test, not a has-metadata test), while `.gco`, +# `.txt` and anything else are NOT, even though Moonraker's own listing +# accepts `.g` and `.gco` as gcode +# - nothing whose name starts with `.` — which is why `gcodes/.thumbs` never +# appears on the screen even though Fluidd shows it +# +# `order` is a comma-separated triple as the screen sends it — +# `name,asc,folder`, `datetime,desc,folder`, `size,asc,folder` — where the third +# token is always `folder` and nothing is keyed off it. Known fields are +# name/filename, datetime (mtime) and size. ANY unrecognised value — including +# "files", "type" and outright nonsense — falls back to name ascending rather +# than erroring, which is nexusp's behaviour and not an accident of this port. +# +# Name ordering is a NATURAL sort here: digit runs compare numerically and text +# compares case-insensitively. nexusp did a plain codepoint sort, which put every +# capitalised name ahead of every lowercase one and ordered `ss_ruin_2`, +# `ss_ruin_10`, `ss_ruin_3` in that order. This is the one deliberate improvement +# on the reference in this file. +# +# `keyword` is a case-insensitive substring match on the name, and it narrows +# `count` as well as the page. +# +# `since` and `before` are accepted and DELIBERATELY IGNORED, because that is +# what nexusp does. Measured, not assumed: a window excluding almost every file +# left `count` unchanged at 92. Implementing them properly would hide files the +# screen has always been shown, which is a worse failure than the one being +# fixed — a file that vanishes from the browser looks like a file that was +# deleted. +# +# `count` is the total AFTER keyword filtering and BEFORE paging, so the screen +# can size its scrollbar. `start` past the end returns an empty page with the +# count intact, and `limit: 0` returns an empty page — also measured. +# +# THUMBNAILS ARE READ OFF THE DISK, NOT OUT OF THE METADATA +# --------------------------------------------------------- +# The first version of this shim passed Moonraker's `thumbnails` through +# untouched, and 66 of 91 files came back thumbnail-less on the screen. The cause +# is not the shim and not the screen: Moonraker lists only the thumbnails the +# SLICER embedded in the gcode, while `nexusp` also rendered its own into +# `.thumbs` and listed those too. Measured — every file has 48/96/195/300 on +# disk, Moonraker lists 48 and 195 for NONE of them, and the 66 files whose +# slicer emitted 100/320 instead of 96/300 have no listed size the screen will +# take. Those 66 are exactly the ones that went blank. +# +# So each file's `thumbnails` is the union of Moonraker's parsed entries and +# every `-x.png` actually present in `.thumbs`. Reporting what exists +# rather than what was parsed is what makes this safe without knowing the +# screen's size-selection rule. +# +# It also GENERATES the two sizes nexusp used to render and Moonraker never +# does — 48x48 and 195x195 — for any file that lacks them, downscaling from the +# largest thumbnail present, once per file, and never by UPSCALING: a 32x32 blown +# up to a 195x195 tile is a blurred mess that reads as a broken thumbnail rather +# than an absent one. Set `generate_thumbnails: False` in `[creality_compat]` to +# turn it off and go back to listing only what exists. +# +# PILLOW IS NOT IN THE MOONRAKER VENV THIS REPO SHIPS +# --------------------------------------------------- +# Upstream Moonraker lists Pillow in requirements.txt, and it is NOT in +# `files/moonraker/moonraker.tar.gz` — the venv the helper actually installs is +# Python 3.8 with apprise, jinja2, ldap3, dbus_fast, inotify_simple and no PIL, +# and `install_moonraker_nginx` only ever runs `git pull` on the source repo. +# So on a clean helper box: +# +# - generation here is off, and says so once in the log rather than throwing +# a traceback per file; +# - Moonraker's OWN metadata.py imports PIL at module scope, so it parses no +# embedded thumbnails either — a freshly uploaded gcode has nothing on disk +# AND nothing parsed, and there is no source image to downscale from. The +# screen shows no thumbnail for it at all. +# +# Installing Pillow into moonraker-env fixes both — the second one for every +# helper user, retired nexusp or not. "Retire Nexusp Backend" offers to do it. +# A directory listing must never fail because an optional renderer is absent, +# so everything above degrades to "list what is on disk" and nothing else. +# +# WHAT IS NOT SHIMMED +# ------------------- +# The gap analysis found four Creality-only methods. The other two stay missing +# on purpose: `server.history.debug.job` returns `{"last_row_id": N}` and is +# internal, and `server.debug.status` dumps nexusp's socket buffers and emits +# MALFORMED JSON (a trailing comma) that strict parsers reject. Neither is +# screen-facing; each was logged exactly once, at connect, with nothing visibly +# broken afterwards. +# +# `printer.info` and `printer.objects.list` are deliberately NOT registered here +# either, even though they answer `-32601 Method not found` for about four +# seconds after a cold boot while the screen polls. They are genuine Moonraker +# methods registered dynamically when klippy connects, and `register_endpoint` +# returns early for an already-registered path when the incoming registration is +# remote — klippy's are remote. A static registration would therefore not +# collide with klippy's later one, it would silently WIN it for the life of the +# process, and a stub answering `printer.info` forever is far worse than a +# four-second gap at boot. +# +# COUPLING, HONESTLY — THREE, NOT TWO +# ----------------------------------- +# 1. `file_manager._list_directory` (private) +# 2. `file_manager._convert_request_path` (private) +# 3. raw SQL against `history.history_table`, Moonraker's own SQL table wrapper +# +# 1 and 2 are private because reimplementing metadata lookup and root resolution +# would be a second source of truth for the thing most likely to drift. Both are +# checked at COMPONENT LOAD, not at request time, and a missing one raises with +# the attribute's name: `install_moonraker_nginx` runs `git checkout master; git +# pull`, so every user's Moonraker is a moving target against a frozen venv, and +# the offline tests below use fakes and can never catch a rename. Failing at +# startup with the missing name is the only warning anyone gets. +# +# The table name for 3 is imported from Moonraker's own history component rather +# than repeated here, so a rename there is an ImportError at load for the same +# reason. + +from __future__ import annotations + +import asyncio +import logging +import os +import re + +from ..common import RequestType, TransportType +from .history import HIST_TABLE + +# Annotation imports +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set + +if TYPE_CHECKING: + from ..confighelper import ConfigHelper + from ..common import WebRequest + +# Measured against nexusp: `.gcode` and nothing else. Moonraker's own +# VALID_GCODE_EXTS is wider (`.g`, `.gco`); matching it here would surface files +# on the screen that have never been there. +GCODE_EXT = ".gcode" + +# The private file_manager methods this component calls. Checked by name at load +# so a Moonraker rename breaks at startup, loudly, instead of the first time +# somebody opens the file browser. +REQUIRED_FM_ATTRS = ("_list_directory", "_convert_request_path") + +# order field -> (item key, default when the key is absent). +# +# MEASURED, at last, from the screen itself: it sends a COMMA-SEPARATED triple — +# `name,asc,folder`, `datetime,desc,folder`, `size,asc,folder`. Field, direction, +# and a third token that is always `folder` (and matches the dirs-always-first +# behaviour this shim already implements unconditionally; no other value has ever +# been observed, so nothing is keyed off it). +# +# That separator was the entire bug behind "sort buttons do nothing": the first +# version split on whitespace only, so `name,asc,folder` arrived as ONE token, +# matched no field, and fell through to the name-ascending default. Silently — +# and that silence is faithful, since falling back rather than erroring on an +# unknown sort is measured nexusp behaviour. The fallback was right; the split +# was wrong. +# +# `name`/`filename`, `datetime` and `size` are nexusp's own vocabulary. The +# aliases below were added while the screen's string was still unknown; they are +# kept because they cost nothing and document what was ruled out. +SORT_FIELDS = { + "name": ("__name__", ""), + "filename": ("__name__", ""), + "file": ("__name__", ""), + "datetime": ("modified", 0.0), + "date": ("modified", 0.0), + "time": ("modified", 0.0), + "modified": ("modified", 0.0), + "mtime": ("modified", 0.0), + "size": ("size", 0), + "filesize": ("size", 0), + "bytes": ("size", 0), +} + +# Sizes nexusp rendered itself, on top of whatever the slicer embedded, and which +# Moonraker therefore never produces. Measured: all 92 pre-swap files have both. +GENERATED_THUMB_SIZES = ((48, 48), (195, 195)) + +# The instance the job_history rows the screen pages through are scoped to. +# Moonraker's own `server.history.list` filters on a bare "default" literal in +# this version, and every row it writes carries that value — so the count and +# the list it sizes cannot disagree. The value is looked up off the history +# component first (see _history_instance) so that stays true if Moonraker ever +# grows a real instance id; this is the fallback, not the answer. +HIST_INSTANCE_FALLBACK = "default" + +# Where Creality's thumbnails live, and how they are named: +# /.thumbs/-x.png. Parsed from the END because a +# gcode stem is full of hyphens and digits ("Cube-4m59s-0.2w-0.1h"). +THUMB_DIR = ".thumbs" +THUMB_RE = re.compile(r"^(?P.+)-(?P\d+)x(?P\d+)\.png$") + +# Bound on the load-time walk that finds `.thumbs` directories to reserve. A +# gcodes root deep or wide enough to exceed this is not a printer this option +# was measured on, and the walk runs before Moonraker serves anything. +MAX_RESERVE_WALK_DIRS = 2000 + +# The screen sends commas (`name,asc,folder`). The rest are accepted because the +# cost of another separator turning up is a sort that silently does nothing. +ORDER_SPLIT = re.compile(r"[\s,;|]+") + +# Natural sort: digit runs compare as numbers, text compares case-insensitively. +# A DELIBERATE DIVERGENCE from nexusp, which did a plain codepoint sort — so it +# put every capitalised name ahead of every lowercase one, and ordered +# `ss_ruin_2`, `ss_ruin_10`, `ss_ruin_3` in that order. Everything else in this +# file matches the reference; this is the one place where being better than it +# was the point. +NATURAL_SPLIT = re.compile(r"(\d+)") + + +def natural_key(name: str) -> Any: + """Sort key: "img2" before "img10", and "apple" beside "Apple". + + Each element is a 3-tuple of the same shape so nothing ever compares an int + against a str — digit runs become `(0, , "")` and text becomes + `(1, 0, )`, which also settles the "do numbers sort before + letters" question consistently rather than by accident. + + The raw name is the final tiebreak, so two names differing only in case get a + stable order instead of an arbitrary one. + """ + parts = NATURAL_SPLIT.split(name) + key = [(0, int(part), "") if index % 2 else (1, 0, part.casefold()) + for index, part in enumerate(parts)] + return key, name + + +class CrealityCompat: + def __init__(self, config: ConfigHelper) -> None: + self.server = config.get_server() + self.log_requests = config.getboolean("log_requests", False) + self.generate_thumbs = config.getboolean("generate_thumbnails", True) + + # Fail at LOAD, with the missing name, rather than the first time the + # screen opens the file browser. See COUPLING, HONESTLY above. + fm = self.server.lookup_component("file_manager") + for attr in REQUIRED_FM_ATTRS: + if not hasattr(fm, attr): + raise self.server.error( + f"creality_compat: file_manager has no '{attr}'. This " + "Moonraker is newer than the component; the touchscreen's " + "file browser needs it. Report it against the helper " + "script rather than editing this file blind.", 500 + ) + + # Pillow, once, here — not per request. Missing PIL must cost one log + # line at startup, not a traceback per file forever. + self._image = self._load_pillow() + # (destination png path) -> already logged. Keyed on the destination + # because it encodes directory, stem and size. In memory only, so a + # repaired filesystem heals at the next Moonraker restart. + self._failed_renders: Set[str] = set() + self._reserved_thumb_dirs: Set[str] = set() + if self.generate_thumbs: + self._reserve_existing_thumb_dirs(fm) + + # WEBSOCKET only, deliberately. `register_endpoint` defaults to every + # transport, which would answer `GET /server/files/directory_ex` over + # HTTP — measured: nexusp returned 404 there, and that 404 is exactly + # what made the method look absent while it was being reverse + # engineered. `server.history.count` follows by symmetry: the screen is + # the only caller and only ever calls it over the websocket. + self.server.register_endpoint( + "/server/files/get_directory_ex", RequestType.GET, + self._handle_directory_ex, transports=TransportType.WEBSOCKET + ) + self.server.register_endpoint( + "/server/history/count", RequestType.GET, + self._handle_history_count, transports=TransportType.WEBSOCKET + ) + + # -- optional dependencies and side channels ---------------------------- + + def _load_pillow(self) -> Optional[Any]: + """PIL.Image, or None with one explanatory line in the log. + + Not a hard dependency: without it this component still serves the + disk-union listing, which is most of the fix. See the PILLOW header + section for why the shipped venv does not have it. + """ + if not self.generate_thumbs: + return None + try: + from PIL import Image + except Exception as why: + self.generate_thumbs = False + logging.info( + "creality_compat: Pillow is not available in Moonraker's " + "virtualenv (%s), so the 48x48 and 195x195 thumbnails the " + "touchscreen expects will not be generated. Thumbnails already " + "on disk are still listed. Note that Moonraker's own metadata " + "parser needs Pillow too, so without it a newly uploaded file " + "has no thumbnail at all. Install it with: " + "/usr/data/moonraker/moonraker-env/bin/python -m pip install Pillow", + why + ) + return None + return Image + + def _reserve_thumb_dir(self, fm: Any, thumb_dir: str) -> None: + """Keep `.thumbs` out of file_manager's inotify watch. + + Writing a PNG into a watched directory fires `notify_filelist_changed`, + and file_manager has no dot-directory filter — so generating thumbnails + inside a listing would broadcast a phantom `create_file` for every PNG + to Fluidd and to the screen. A reserved path is skipped both by the + initial scan and by the directory-create handler, and read access is + left on so the thumbnails are still served over HTTP. + + This must happen BEFORE the directory is scanned or created, which is + why the existing ones are reserved at load (file_manager's initial scan + runs in its `component_init`, after every component is constructed) and + a new one is reserved before it is made. Both `add_reserved_path` and + `get_directory` are public API; a failure here is cosmetic, so it is + logged and swallowed rather than raised. + """ + if thumb_dir in self._reserved_thumb_dirs: + return + self._reserved_thumb_dirs.add(thumb_dir) + try: + fm.add_reserved_path(f"creality_compat:{thumb_dir}", thumb_dir, True) + except Exception: + logging.exception( + "creality_compat: could not reserve %s; thumbnail writes there " + "will emit spurious filelist notifications", thumb_dir + ) + + def _ensure_thumb_dir(self, dir_path: str) -> bool: + """Reserve `/.thumbs`, then make sure it exists. Order matters. + + Reserving first is what stops the directory-create event from starting a + watch on it, which is what stops every PNG written afterwards from + broadcasting a phantom `create_file`. Called only when there is actually + something to render, so a directory of thumbnail-less files never grows + an empty `.thumbs`. + """ + thumb_dir = os.path.join(dir_path, THUMB_DIR) + self._reserve_thumb_dir( + self.server.lookup_component("file_manager"), thumb_dir) + if os.path.isdir(thumb_dir): + return True + try: + os.makedirs(thumb_dir) + except OSError as why: + logging.warning( + "creality_compat: cannot create %s (%s); listing only the " + "thumbnails already on disk", thumb_dir, why + ) + return False + return True + + def _reserve_existing_thumb_dirs(self, fm: Any) -> None: + try: + gcode_root = fm.get_directory("gcodes") + except Exception: + gcode_root = "" + if not gcode_root or not os.path.isdir(gcode_root): + return + seen = 0 + for dir_path, subdirs, _ in os.walk(gcode_root): + seen += 1 + if seen > MAX_RESERVE_WALK_DIRS: + logging.info( + "creality_compat: stopped reserving .thumbs directories " + "after %d directories; deeper ones will emit spurious " + "filelist notifications when a thumbnail is written", + MAX_RESERVE_WALK_DIRS + ) + return + if THUMB_DIR in subdirs: + subdirs.remove(THUMB_DIR) + self._reserve_thumb_dir(fm, os.path.join(dir_path, THUMB_DIR)) + + def _history_instance(self, history: Any) -> str: + """The instance id `server.history.list` scopes to. + + Derived rather than hardcoded so the docstring on `_handle_history_count` + stays true: a count that disagrees with the list it sizes is exactly the + plausible-wrong-answer failure retiring nexusp exists to kill. Today's + Moonraker has no such attribute and uses a bare "default" literal in + both the insert and the list query, so the fallback is the answer — but + it is the fallback. + """ + instance = getattr(history, "instance_id", None) + if isinstance(instance, str) and instance: + return instance + return HIST_INSTANCE_FALLBACK + + # -- server.files.get_directory_ex -------------------------------------- + + def _require(self, web_request: WebRequest, *names: str) -> None: + """Reject a missing argument the way nexusp does. + + `path` is left to Moonraker's own getter, whose "No data for argument: + path" happens to match nexusp's wording exactly. The other three get the + bare "Invalid parameter" — unhelpful, but it is what the screen has + always received and what any future debugger will find in the capture. + """ + args = web_request.get_args() + for name in names: + if name not in args: + raise self.server.error("Invalid parameter", 400) + + def _visible(self, name: str, is_dir: bool) -> bool: + if name.startswith("."): + return False + return is_dir or name.lower().endswith(GCODE_EXT) + + def _disk_thumbnails(self, dir_path: str) -> Dict[str, List[Dict[str, Any]]]: + """Every PNG actually sitting in `/.thumbs`, keyed by gcode stem. + + Moonraker lists only the thumbnails the SLICER embedded in the gcode, and + `nexusp` rendered its own on top of those. Measured on the reference unit: + all 92 files have 48/96/195/300 on disk, and Moonraker lists 48 and 195 + for none of them — while the 66 files whose slicer emitted 100/320 have + no listed size the screen will accept at all. Those 66 are exactly the + files that came back thumbnail-less after the swap. + + So the shim reports what EXISTS rather than what was parsed. The listing + becomes a superset of both daemons' — which is what makes it safe without + knowing the screen's size-selection rule, since whatever it looks for is + now there. + """ + found: Dict[str, List[Dict[str, Any]]] = {} + thumb_dir = os.path.join(dir_path, THUMB_DIR) + try: + names = os.listdir(thumb_dir) + except OSError: + # No .thumbs here at all. Normal for every root except gcodes. + return found + for name in names: + match = THUMB_RE.match(name) + if match is None: + continue + try: + size = os.path.getsize(os.path.join(thumb_dir, name)) + except OSError: + continue + found.setdefault(match.group("stem"), []).append({ + "width": int(match.group("w")), + "height": int(match.group("h")), + "size": size, + "relative_path": f"{THUMB_DIR}/{name}", + }) + return found + + def _generate_missing( + self, dir_path: str, stem: str, have: List[Dict[str, Any]] + ) -> List[Dict[str, Any]]: + """Render the sizes nexusp used to render, from the biggest one present. + + Only for files that lack them, only the two sizes in + GENERATED_THUMB_SIZES, and only ever by DOWNSCALING — upscaling a 32x32 + into a 195x195 tile produces a blurred mess that looks like a broken + thumbnail rather than an absent one, which is worse than blank. + + A file with nothing on disk and nothing parsed gets nothing: there is no + source image, and inventing one is not on the table. On a box without + Pillow that is every freshly uploaded file, because Moonraker's metadata + parser needs Pillow too — see the header. + + Runs in a worker thread (see `_decorate_page`), never on the event loop: + LANCZOS resizes plus flash writes for a whole page would otherwise block + the loop that also serves klippy and Fluidd, during a print. + """ + if not self.generate_thumbs or self._image is None: + return [] + source = None + for thumb in have: + if source is None or thumb["width"] * thumb["height"] > \ + source["width"] * source["height"]: + source = thumb + if source is None: + return [] + wanted = [ + (width, height) for width, height in GENERATED_THUMB_SIZES + if not any(t["width"] == width and t["height"] == height for t in have) + and source["width"] >= width and source["height"] >= height + ] + if not wanted or not self._ensure_thumb_dir(dir_path): + return [] + made: List[Dict[str, Any]] = [] + for width, height in wanted: + name = f"{stem}-{width}x{height}.png" + dest = os.path.join(dir_path, THUMB_DIR, name) + # One attempt per destination per process. Retrying a render that + # has already failed means a traceback on every listing forever; + # a Moonraker restart clears the set, so a repaired filesystem heals + # on its own. + if dest in self._failed_renders: + continue + try: + with self._image.open( + os.path.join(dir_path, source["relative_path"]) + ) as im: + im.convert("RGBA").resize( + (width, height), self._image.LANCZOS).save(dest) + made.append({ + "width": width, "height": height, + "size": os.path.getsize(dest), + "relative_path": f"{THUMB_DIR}/{name}", + }) + except Exception as why: + # A directory listing must not fail because one PNG would not + # scale. One warning per destination, then silence. + self._failed_renders.add(dest) + logging.warning( + "creality_compat: could not render %s (%s); not trying " + "again until Moonraker restarts", dest, why + ) + return made + + def _merge_thumbnails( + self, entry: Dict[str, Any], on_disk: Dict[str, List[Dict[str, Any]]], + dir_path: str + ) -> None: + """Add the on-disk thumbnails this file has that the metadata omits. + + Moonraker's own entries are kept as-is and win on duplicate dimensions — + it read them out of the gcode, which is the better provenance. Sorted by + area so the result is deterministic; nexusp's own order was its database's + insertion order and the screen evidently searches the list rather than + taking the first, since it accepted several different orders. + """ + stem = os.path.splitext(entry.get("filename", ""))[0] + thumbs = list(entry.get("thumbnails") or []) + have = {(t.get("width"), t.get("height")) for t in thumbs} + thumbs.extend(t for t in on_disk.get(stem, []) + if (t["width"], t["height"]) not in have) + thumbs.extend(self._generate_missing(dir_path, stem, thumbs)) + if not thumbs: + return + entry["thumbnails"] = sorted(thumbs, key=lambda t: t["width"] * t["height"]) + + def _decorate_page(self, dir_path: str, page: List[Dict[str, Any]]) -> None: + """Thumbnail work for the page only, off the event loop. + + Deliberately AFTER the slice: sorting reads `filename`, `modified` and + `size` and never `thumbnails`, so nothing above needs this. Doing it + before the slice meant the first browse of a 91-file directory with no + `.thumbs` was up to 182 LANCZOS resizes plus flash writes inside one + request — during a print, on the loop serving klippy and Fluidd. + """ + files = [item for item in page if item.get("type") == "f"] + if not files: + return + on_disk = self._disk_thumbnails(dir_path) + for entry in files: + self._merge_thumbnails(entry, on_disk, dir_path) + + def _sorted(self, items: List[Dict[str, Any]], order: str) -> List[Dict[str, Any]]: + # Case-folded, and split on commas/semicolons as well as whitespace: the + # screen sends `name,asc,folder`, and the cost of another separator + # turning up is a sort that silently does nothing. "desc" anywhere after + # the field means descending. + tokens = [t for t in ORDER_SPLIT.split(order.strip().lower()) if t] + field = tokens[0] if tokens else "name" + reverse = "desc" in tokens[1:] + key, default = SORT_FIELDS.get(field, SORT_FIELDS["name"]) + if key == "__name__": + def sort_key(item: Dict[str, Any]) -> Any: + return natural_key( + item.get("filename") or item.get("dirname") or "") + else: + def sort_key(item: Dict[str, Any]) -> Any: + value = item.get(key) + return default if value is None else value + return sorted(items, key=sort_key, reverse=reverse) + + async def _handle_directory_ex(self, web_request: WebRequest) -> Dict[str, Any]: + self._require(web_request, "start", "limit", "order") + path = web_request.get_str("path") + start = web_request.get_int("start") + limit = web_request.get_int("limit") + order = web_request.get_str("order") + keyword = web_request.get_str("keyword", "").lower() + # A negative offset would wrap the slice and hand back the END of the + # listing, which reads as data corruption rather than a bad request. + start = max(start, 0) + + fm = self.server.lookup_component("file_manager") + root, dir_path = fm._convert_request_path(path) + listing = fm._list_directory(dir_path, root, True) + + if self.log_requests: + # The screen is the only caller and it is closed source; this line is + # the only way to learn what it actually sends. Cheap, and off by + # default once the questions it answers are answered. + logging.info("creality_compat: get_directory_ex %s", web_request.get_args()) + + dirs: List[Dict[str, Any]] = [] + files: List[Dict[str, Any]] = [] + for entry in listing["dirs"]: + name = entry.get("dirname", "") + if self._visible(name, True) and keyword in name.lower(): + dirs.append(dict(entry, type="d")) + for entry in listing["files"]: + name = entry.get("filename", "") + if self._visible(name, False) and keyword in name.lower(): + files.append(dict(entry, type="f")) + + # Directories first, always, whatever the sort — measured, and it is what + # makes the screen's "up one level" row sit where the user expects. + items = self._sorted(dirs, order) + self._sorted(files, order) + count = len(items) + page = items[start:start + limit] if limit > 0 else [] + if page: + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, self._decorate_page, dir_path, page) + + root_info = dict(listing.get("root_info", {})) + # nexusp carries `path` here and Moonraker does not. The screen is the + # only reader and it has always been given one. + root_info.setdefault("name", root) + root_info["path"] = path + return {"items": page, "count": count, "root_info": root_info} + + # -- server.history.count ----------------------------------------------- + + async def _handle_history_count(self, web_request: WebRequest) -> Dict[str, Any]: + """Rows in job_history for this instance. + + nexusp takes no arguments here and ignores any that are sent; the + instance filter matches what `server.history.list` already scopes to, so + the number the screen shows cannot disagree with the list it pages. + """ + history = self.server.lookup_component("history") + cursor = await history.history_table.execute( + f"SELECT COUNT(*) FROM {HIST_TABLE} WHERE instance_id = ?", + (self._history_instance(history),) + ) + row = await cursor.fetchone() + return {"count": int(row[0]) if row is not None else 0} + + +def load_component(config: ConfigHelper) -> CrealityCompat: + return CrealityCompat(config) diff --git a/files/moonraker/creality-compat/merge_job_history.py b/files/moonraker/creality-compat/merge_job_history.py new file mode 100644 index 0000000..0fbb6d7 --- /dev/null +++ b/files/moonraker/creality-compat/merge_job_history.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +# merge_job_history.py — fold one of the K1C 2025's two print histories into the +# other. Runs ON the printer, with both daemons stopped. Not a service, not +# scheduled; "Retire Nexusp Backend" calls it once in each direction. +# +# THE PIPELINE +# ------------ +# +# source db (read-only) target db (written) +# | | +# | job_history rows | job_history rows +# v v +# +-------------------------------------------------------+ +# | classify each source row against the target | +# | | +# | status == in_progress -> SKIPPED | +# | (filename, start_time +/-120s) matches -> DUPLICATE | +# | otherwise -> INSERT | +# +-------------------------------------------------------+ +# | | | +# | listed | counted | listed +# v v v +# dry run prints all three, then stops unless --apply +# | +# | --apply: no Moonraker or nexusp process alive +# v +# backup the target -> .bak-merge- +# | +# +--> insert the new rows (ids continue after the target's) +# +--> job_totals: per-column max() of target vs source +# +# WHAT THIS IS FOR +# ---------------- +# The 2025 runs two Moonrakers against one Klipper: Creality's `nexusp` (the +# touchscreen's backend, :7125) and the helper script's real one (:7126). They +# share `-d /usr/data/printer_data` — one gcode directory, one klippy socket — +# but Creality namespaced the databases, so each keeps its OWN print history: +# +# printer_data/database/nexusp-sql.db <- what the touchscreen shows +# printer_data/database/moonraker-sql.db <- what Fluidd shows +# +# Every print since the helper stack was installed is therefore written down +# twice, and everything BEFORE that install exists only in nexusp's copy. When +# real Moonraker takes over :7125 and nexusp is retired, the screen starts +# reading Fluidd's database — so without this the printer's history silently +# begins on the day the helper was installed. On the reference unit that was +# 22 of 42 rows that exist nowhere else, including two prints the helper's +# Moonraker had missed entirely. +# +# BOTH DIRECTIONS, AND WHY RESTORE NEEDS ONE TOO +# ---------------------------------------------- +# `--direction to-moonraker` (the default) is retirement: nexusp's rows into +# Moonraker's database. `--direction to-nexusp` is the restore, and it is not +# optional. While nexusp is retired the touchscreen reads and WRITES +# moonraker-sql.db; handing it back a nexusp-sql.db frozen at the retirement +# date would make every print made in between vanish from the screen, with no +# warning and no reason for the user to connect the two events. Same dedupe, +# same backup, same second-run no-op — only the two paths swap. +# +# This is NOT a sync tool. It is one-shot per direction, for the swap window. +# Running it twice is harmless (the dedupe catches everything it already +# inserted), but it has no business existing as a cron job. +# +# THE DEDUPE, AND WHY IT IS FUZZY +# ------------------------------- +# Both daemons watch the same klippy, so a print recorded by both produces two +# rows describing one physical job. They are NOT byte-identical: each daemon +# stamps its own `start_time` when it notices the state change, and on the +# reference unit those land ~0.2-1.1 s apart. So the match key is (filename, +# start_time within TOLERANCE_S) rather than equality. +# +# 120 s is deliberately far wider than the observed skew. The input that could +# defeat it is not two unrelated prints — it is a failed print and its immediate +# retry, same filename, minutes apart. Measured on the reference unit, the +# closest such pair is over 600 s apart, so the window keeps 5x margin against +# its own worst realistic case. +# +# JOB_TOTALS ARE NOT ADDITIVE +# --------------------------- +# `job_totals` holds lifetime counters, and the two rowsets OVERLAP, so summing +# them would double-count every shared print. It is also not a simple function of +# job_history: nexusp reports total_jobs=184 while holding 42 rows, because the +# counter survives rows that were pruned. So the merge takes max() PER COLUMN — +# `total` and `maximum` describe different things (a lifetime sum and a single +# job's record), and the row holding the larger sum is not guaranteed to hold the +# larger record. Nothing is invented; each column wins on its own merits. Expect +# Fluidd's "total jobs" to jump from double digits to the machine's real lifetime +# figure. That is the correct number; it only looks wrong because Fluidd has been +# reporting a post-install subset all along. +# +# SAFETY +# ------ +# - Dry run by default. `--apply` is required to write anything. +# - Refuses to run while a Moonraker or nexusp process is alive: Moonraker caches +# `job_totals` in memory and flushes its stale copy back at the next print, +# straight over the merge. +# - That check is POINT IN TIME. Anything that can restart Moonraker behind your +# back — a supervisor, a watchdog, a cron entry — must be stopped first, not +# just the daemon, or it can start one in the gap between the check passing and +# the write landing. This repo ships no such watchdog, so upstream this is a +# warning rather than a step; a fork that adds one owns disarming it. +# - Backs the target up next to itself before the first write, and prints the +# command that restores it. That file is the entire rollback story. +# - Does NOT renumber job_id. Inserted rows take ids after the target's existing +# ones, so id order no longer matches time order — deliberately. Nothing joins +# on job_id and every surface sorts by start_time, whereas renumbering rewrites +# the identity of rows a client may already be holding: a Fluidd tab left open +# across the merge would delete by an id that now names a different print. + +import argparse +import os +import shutil +import sqlite3 +import sys +import time + +TOLERANCE_S = 120.0 + +DB_FOLDER = "/usr/data/printer_data/database" +MOONRAKER_DB = os.path.join(DB_FOLDER, "moonraker-sql.db") +NEXUSP_DB = os.path.join(DB_FOLDER, "nexusp-sql.db") + +# direction -> (source, target). Retirement moves the screen's history into +# Moonraker's database; restore moves everything printed while retired back. +DIRECTIONS = { + "to-moonraker": (NEXUSP_DB, MOONRAKER_DB), + "to-nexusp": (MOONRAKER_DB, NEXUSP_DB), +} + +# Column order is identical in both schemas — verified on the reference unit, +# where nexusp declares metadata/auxiliary_data as TEXT and moonraker declares +# them `pyjson`. That difference is cosmetic: sqlite type names are advisory and +# both store the same JSON text, which is what Moonraker's pyjson converter +# expects to read back. +COLUMNS = ( + "user", "filename", "status", "start_time", "end_time", + "print_duration", "total_duration", "filament_used", + "metadata", "auxiliary_data", "instance_id", +) + +MAX_FIELDS = ("total_jobs", "total_time", "total_print_time", + "total_filament_used", "longest_job", "longest_print") + + +def moonraker_running(): + """True if anything that looks like Moonraker holds a PID right now. + + /proc scan rather than pgrep: busybox ps on this board truncates the command + line at a width that hides moonraker.py behind the venv python path. + """ + if not os.path.isdir("/proc"): + # Not the printer — a dry-run rehearsal against copied databases on a + # workstation. There is no daemon here to collide with. + return None + for pid in os.listdir("/proc"): + if not pid.isdigit(): + continue + try: + with open("/proc/%s/cmdline" % pid, "rb") as fh: + cmd = fh.read().decode("utf-8", "replace") + except (IOError, OSError): + continue + if "moonraker.py" in cmd or "/bin/nexusp" in cmd: + return cmd.replace("\0", " ").strip() + return None + + +def load_jobs(db): + con = sqlite3.connect("file:%s?mode=ro" % db, uri=True) + con.row_factory = sqlite3.Row + rows = con.execute("select * from job_history order by start_time").fetchall() + totals = con.execute("select * from job_totals").fetchall() + con.close() + return rows, totals + + +def is_duplicate(row, targets): + for t in targets: + if row["filename"] == t["filename"] and \ + abs(row["start_time"] - t["start_time"]) <= TOLERANCE_S: + return t + return None + + +def classify(source_rows, target_rows): + """Split the source into (insert, duplicate, skipped) buckets. + + `skipped` is its own bucket rather than a third kind of duplicate: an + in_progress row is the live job, and copying a row whose end_time is still + NULL leaves a job that never completes. Filing it under "duplicate" would + report a DROP as a no-op, which is the one thing a dry run must not do. + """ + new, dupes, skipped = [], [], [] + for row in source_rows: + if row["status"] == "in_progress": + skipped.append(row) + continue + match = is_duplicate(row, target_rows) + (dupes if match else new).append((row, match)) + return new, dupes, skipped + + +def merge_totals(target_totals, source_totals): + """Per-column max() of the two totals tables, keyed by (provider, field, instance). + + Per COLUMN, not per row: taking whole rows would let a genuine longest-print + figure be discarded because its row happened to lose on `total`. + """ + merged = {} + for rowset in (target_totals, source_totals): + for t in rowset: + key = (t["provider"], t["field"], t["instance_id"]) + prev = merged.get(key) + if prev is None: + merged[key] = {"maximum": t["maximum"], "total": t["total"]} + continue + for col in ("maximum", "total"): + incoming = t[col] + if incoming is None: + continue + if prev[col] is None or incoming > prev[col]: + prev[col] = incoming + return merged + + +def main(): + ap = argparse.ArgumentParser( + description="fold one K1C 2025 print history into the other") + ap.add_argument("--direction", choices=sorted(DIRECTIONS), default="to-moonraker", + help="to-moonraker when retiring nexusp (default), " + "to-nexusp when restoring it") + ap.add_argument("--into", default=None, + help="override the target database, written to") + ap.add_argument("--source", default=None, + help="override the source database, read only") + ap.add_argument("--apply", action="store_true", help="actually write") + ap.add_argument("--force", action="store_true", + help="write even with a daemon alive — it will then overwrite " + "the merged job_totals from its stale in-memory copy") + args = ap.parse_args() + + default_source, default_into = DIRECTIONS[args.direction] + source_db = args.source or default_source + into_db = args.into or default_into + + for path in (into_db, source_db): + if not os.path.exists(path): + sys.exit("missing database: %s" % path) + + target_rows, target_totals = load_jobs(into_db) + source_rows, source_totals = load_jobs(source_db) + + new, dupes, skipped = classify(source_rows, target_rows) + + def when(t): + return time.strftime("%Y-%m-%d %H:%M", time.localtime(t)) + + def listing(rows): + for row in rows: + print(" %s %-44s %s" % (when(row["start_time"]), + (row["filename"] or "")[:44], row["status"])) + + print("direction: %s" % args.direction) + print("target %s: %d jobs" % (into_db, len(target_rows))) + print("source %s: %d jobs" % (source_db, len(source_rows))) + print("duplicate (skipped): %d" % len(dupes)) + print("in_progress (NOT copied): %d" % len(skipped)) + listing(skipped) + print("to insert: %d" % len(new)) + listing([row for row, _ in new]) + + merged_totals = merge_totals(target_totals, source_totals) + print("job_totals after merge:") + for (provider, field, inst), t in sorted(merged_totals.items()): + if field in MAX_FIELDS: + print(" %-22s total=%s maximum=%s" % (field, t["total"], t["maximum"])) + + if not args.apply: + print("\ndry run — nothing written. re-run with --apply") + return + + alive = moonraker_running() + if alive and not args.force: + sys.exit("refusing: a daemon is still alive -> %s\n" + "stop it before merging; its cached job_totals would land on " + "top of the merge at the next print." % alive) + + backup = "%s.bak-merge-%s" % (into_db, time.strftime("%Y%m%d_%H%M%S")) + shutil.copy2(into_db, backup) + print("\nbackup: %s" % backup) + print("rollback: stop Moonraker, cp %s %s, restart it" % (backup, into_db)) + + con = sqlite3.connect(into_db) + con.row_factory = sqlite3.Row + try: + with con: + for row, _ in new: + con.execute( + "insert into job_history (%s) values (%s)" + % (",".join(COLUMNS), ",".join("?" * len(COLUMNS))), + tuple(row[c] for c in COLUMNS)) + for (provider, field, inst), t in merged_totals.items(): + con.execute( + "insert or replace into job_totals " + "(provider, field, maximum, total, instance_id) values (?,?,?,?,?)", + (provider, field, t["maximum"], t["total"], inst)) + total = con.execute("select count(*) from job_history").fetchone()[0] + print("merged: %d jobs in %s" % (total, into_db)) + finally: + con.close() + + +if __name__ == "__main__": + main() diff --git a/files/moonraker/creality-compat/test_creality_compat.py b/files/moonraker/creality-compat/test_creality_compat.py new file mode 100644 index 0000000..92d9bb4 --- /dev/null +++ b/files/moonraker/creality-compat/test_creality_compat.py @@ -0,0 +1,945 @@ +#!/usr/bin/env python3 +"""Offline checks for creality_compat.py — no printer, no Moonraker, no network. + +Standalone, no conftest and no fixtures beyond this file: + + python3 -m pytest -q test_creality_compat.py + +WHY THIS FILE EXISTS +-------------------- +The component is a compatibility shim, and a shim's only job is to behave like +the thing it replaces. The thing it replaces — Creality's `nexusp` — is disabled +on any machine where this component runs, so "does it still behave right?" stops +being answerable by experiment the moment someone edits it. These tests are the +executable record of what the measurements found, and once a user has retired +nexusp they cannot be re-derived without reviving it on a spare port. + +The rules being pinned are all counter-intuitive enough that a well-meaning +edit would break them: + + - directories sort BEFORE files, whatever the sort field + - `.gco` and `.g` are hidden even though Moonraker itself calls them gcode + - an EMPTY `.gcode` is shown, so the test is the extension, not the metadata + - dotfiles are hidden, which is the only reason `gcodes/.thumbs` never + appeared on the screen + - `since`/`before` are accepted and IGNORED on purpose — implementing them + would hide files the screen has always been shown + - an unrecognised `order` falls back to name ascending rather than erroring + - `count` is the total after filtering and before paging + - thumbnail work happens AFTER the page is sliced, so a 91-file directory + does not do 182 resizes to serve 20 rows + +Nothing here touches Moonraker. The component is loaded into a synthetic package +with `..common` and `.history` stubbed, and `file_manager`/`history` are fakes +that return exactly the shapes the real ones do — which is the point: if +Moonraker renames `_list_directory`, these tests keep passing and the printer +breaks. That gap is why the component checks for those attributes at LOAD time; +the check itself is pinned below. + +⚠ Add cases as `test_*` functions. A file named test_*.py whose assertions live +in a hand-rolled runner gets collected and runs nothing while reporting green. +""" +import asyncio +import importlib.machinery +import importlib.util +import os +import sys +import types + +import pytest + +REPO = os.path.dirname(os.path.abspath(__file__)) +PKG = "k1c_compat_undertest" + + +def _load(): + """Load creality_compat.py as a package member, with Moonraker stubbed. + + TWO package levels, not one. The component says `from ..common import + RequestType` because on the printer it lives at + `moonraker/components/creality_compat.py` — so it has to be loaded as + `.components.creality_compat` for that `..` to resolve. Flattening it + one level up fails with "attempted relative import beyond top-level + package", which is a confusing way to be told the fixture is wrong rather + than the component. + + `.components.history` is stubbed too: the component imports HIST_TABLE + from Moonraker's own history component rather than repeating the table name, + so that a rename upstream is an ImportError at load. + """ + name = f"{PKG}.components.creality_compat" + if name in sys.modules: + return sys.modules[name] + + class RequestType: + GET = "GET" + + class TransportType: + HTTP = "HTTP" + WEBSOCKET = "WEBSOCKET" + + root = types.ModuleType(PKG) + root.__path__ = [REPO] + components = types.ModuleType(f"{PKG}.components") + components.__path__ = [REPO] + common = types.ModuleType(f"{PKG}.common") + common.RequestType = RequestType + common.TransportType = TransportType + history = types.ModuleType(f"{PKG}.components.history") + history.HIST_TABLE = "job_history" + root.components = components + root.common = common + components.history = history + sys.modules[PKG] = root + sys.modules[f"{PKG}.components"] = components + sys.modules[f"{PKG}.common"] = common + sys.modules[f"{PKG}.components.history"] = history + + path = os.path.join(REPO, "creality_compat.py") + spec = importlib.util.spec_from_file_location( + name, path, loader=importlib.machinery.SourceFileLoader(name, path)) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +cc = _load() +WEBSOCKET = sys.modules[f"{PKG}.common"].TransportType.WEBSOCKET + + +class ServerError(Exception): + def __init__(self, message, status_code=400): + super().__init__(message) + self.message = message + self.status_code = status_code + + +class FakeCursor: + def __init__(self, row): + self._row = row + + async def fetchone(self): + return self._row + + +class FakeTable: + def __init__(self, row): + self._row = row + self.queries = [] + + async def execute(self, sql, params=()): + self.queries.append((sql, params)) + return FakeCursor(self._row) + + +class FakeHistory: + def __init__(self, count, instance_id=None): + self.history_table = FakeTable((count,)) + if instance_id is not None: + self.instance_id = instance_id + + +class FakeFileManager: + """Returns exactly what `_list_directory(..., extended=True)` returns.""" + + def __init__(self, dirs=(), files=(), root="gcodes", permissions="rw", + disk_root=None): + self.dirs, self.files = list(dirs), list(files) + self.root, self.permissions = root, permissions + self.converted = [] + self.reserved = {} + # When set, `_convert_request_path` hands back a REAL directory so the + # component's `.thumbs` scan has something to read. + self.disk_root = disk_root + + def _convert_request_path(self, path): + root = path.strip("/").split("/", 1)[0] + if root != self.root: + raise ServerError(f"Invalid root path ({root})") + self.converted.append(path) + return self.root, self.disk_root or ("/fake/" + path) + + def _list_directory(self, dir_path, root, extended=False): + return { + "dirs": [dict(d) for d in self.dirs], + "files": [dict(f) for f in self.files], + "disk_usage": {"total": 1, "used": 0, "free": 1}, + "root_info": {"name": root, "permissions": self.permissions}, + } + + def get_directory(self, root="gcodes"): + return self.disk_root or "" + + def add_reserved_path(self, name, res_path, read_access=True): + if name in self.reserved: + return False + self.reserved[name] = (str(res_path), read_access) + return True + + +class FakeServer: + def __init__(self, components): + self.components = components + self.registered = [] + + def register_endpoint(self, endpoint, request_types, callback, **kwargs): + self.registered.append((endpoint, kwargs.get("transports"))) + + def lookup_component(self, name): + return self.components[name] + + def error(self, message, status_code=400): + return ServerError(message, status_code) + + +class FakeConfig: + def __init__(self, server, **options): + self._server = server + # Defaults match moonraker.conf on the box: tracing off, generation on. + self.options = options + + def get_server(self): + return self._server + + def getboolean(self, key, default=None): + return self.options.get(key, default) + + +class FakeWebRequest: + def __init__(self, **args): + self.args = args + + def get_args(self): + return self.args + + def _get(self, key, default, cast): + if key not in self.args: + if default is _MISSING: + raise ServerError(f"No data for argument: {key}") + return default + return cast(self.args[key]) + + def get_str(self, key, default=None): + return self._get(key, _MISSING if default is None else default, str) + + def get_int(self, key, default=None): + return self._get(key, _MISSING if default is None else default, int) + + +class _Missing: + pass + + +_MISSING = _Missing() + + +@pytest.fixture(autouse=True) +def pil_absent_by_default(monkeypatch): + """Pin the workstation to the printer's shipped state: no Pillow. + + `files/moonraker/moonraker.tar.gz` has no PIL, so that is the default a + helper user gets and the default these tests run against. Setting the + sys.modules entry to None is what makes `from PIL import Image` raise. + Tests about generation ask for `fake_pil`, which runs after this and wins. + """ + monkeypatch.setitem(sys.modules, "PIL", None) + monkeypatch.setitem(sys.modules, "PIL.Image", None) + + +def entry(name, is_dir=False, modified=100.0, size=10): + key = "dirname" if is_dir else "filename" + return {key: name, "modified": modified, "size": size, "permissions": "rw"} + + +def build(dirs=(), files=(), history=42, root="gcodes", disk_root=None, + instance_id=None, **options): + fm = FakeFileManager(dirs=dirs, files=files, root=root, disk_root=disk_root) + server = FakeServer({"file_manager": fm, + "history": FakeHistory(history, instance_id)}) + return cc.CrealityCompat(FakeConfig(server, **options)), fm, server + + +def run(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +def call(shim, **args): + args.setdefault("path", "gcodes") + args.setdefault("start", 0) + args.setdefault("limit", 100) + args.setdefault("order", "name") + return run(shim._handle_directory_ex(FakeWebRequest(**args))) + + +def names(result): + return [(i["type"], i.get("filename") or i.get("dirname")) for i in result["items"]] + + +# -------------------------------------------------------------------------- +# Registration and load-time guards +# -------------------------------------------------------------------------- + +def test_it_registers_exactly_the_two_missing_methods(): + _, _, server = build() + assert [ep for ep, _ in server.registered] == [ + "/server/files/get_directory_ex", "/server/history/count"] + + +def test_both_methods_are_websocket_only(): + """`GET /server/files/directory_ex` returned 404 on nexusp — measured, and + that 404 is exactly what made the method look absent. Registering on the + default transports would answer it over HTTP instead.""" + _, _, server = build() + assert [t for _, t in server.registered] == [WEBSOCKET, WEBSOCKET] + + +@pytest.mark.parametrize("missing", ["_list_directory", "_convert_request_path"]) +def test_a_renamed_file_manager_method_fails_at_load_by_name(missing): + """The only warning anyone gets. `install_moonraker_nginx` runs `git + checkout master; git pull`, so every user's Moonraker is a moving target, + and the fakes in this file can never catch a rename. Failing at startup with + the attribute's name beats failing the first time somebody opens the file + browser.""" + def gone(self): + raise AttributeError(missing) + + renamed = type("RenamedFileManager", (FakeFileManager,), + {missing: property(gone)}) + server = FakeServer({"file_manager": renamed(), "history": FakeHistory(0)}) + with pytest.raises(ServerError) as exc: + cc.CrealityCompat(FakeConfig(server)) + assert missing in exc.value.message + + +# -------------------------------------------------------------------------- +# What is visible +# -------------------------------------------------------------------------- + +def test_directories_sort_before_files_whatever_the_order(): + shim, _, _ = build(dirs=[entry("zzz_dir", True, modified=1.0, size=1)], + files=[entry("aaa.gcode", modified=999.0, size=999)]) + for order in ("name", "name desc", "datetime desc", "size desc", "size"): + assert names(call(shim, order=order))[0][0] == "d", order + + +def test_dotfiles_and_dotdirs_are_hidden(): + """The only reason `gcodes/.thumbs` never appeared on the touchscreen.""" + shim, _, _ = build(dirs=[entry(".thumbs", True), entry("keep", True)], + files=[entry(".hidden.gcode"), entry("keep.gcode")]) + assert names(call(shim)) == [("d", "keep"), ("f", "keep.gcode")] + + +def test_only_dot_gcode_counts_as_a_file(): + """`.g` and `.gco` ARE gcode to Moonraker and are NOT listed here. + + Measured against nexusp: a `.gco` file it could see was not returned. Being + more permissive would put files on the screen that have never been there. + """ + shim, _, _ = build(files=[entry("a.gcode"), entry("b.gco"), entry("c.g"), + entry("d.txt"), entry("e.GCODE")]) + assert names(call(shim)) == [("f", "a.gcode"), ("f", "e.GCODE")] + + +def test_an_empty_gcode_is_listed(): + """The test is the extension, not the metadata — measured with `touch`.""" + shim, _, _ = build(files=[entry("empty.gcode", size=0)]) + assert names(call(shim)) == [("f", "empty.gcode")] + + +# -------------------------------------------------------------------------- +# Ordering +# -------------------------------------------------------------------------- + +def test_name_order_is_case_insensitive(): + """A DELIBERATE divergence from nexusp, which sorted by codepoint and so put + every capitalised name ahead of every lowercase one — `Assembly`, `BM`, + `apple`, `ss_low`.""" + shim, _, _ = build(files=[entry("ss_low.gcode"), entry("Assembly.gcode"), + entry("apple.gcode"), entry("BM.gcode")]) + assert [n for _, n in names(call(shim, order="name"))] == [ + "apple.gcode", "Assembly.gcode", "BM.gcode", "ss_low.gcode"] + + +def test_digit_runs_sort_numerically(): + """`ss_ruin_2` before `ss_ruin_10`. Codepoint order put 10 before 2, which + is what a directory of numbered parts looks wrong in.""" + shim, _, _ = build(files=[entry("ss_ruin_10.gcode"), entry("ss_ruin_2.gcode"), + entry("ss_ruin_3.gcode"), entry("ss_ruin_1.gcode")]) + assert [n for _, n in names(call(shim, order="name"))] == [ + "ss_ruin_1.gcode", "ss_ruin_2.gcode", "ss_ruin_3.gcode", + "ss_ruin_10.gcode"] + + +def test_a_number_never_compares_against_a_letter(): + """The key is built so digit and text chunks are never compared directly — + otherwise Python raises TypeError partway through a sort and the file + browser 500s on one awkward filename.""" + shim, _, _ = build(files=[entry("2.gcode"), entry("a2.gcode"), + entry("2a.gcode"), entry("a.gcode")]) + got = [n for _, n in names(call(shim, order="name"))] + assert len(got) == 4 and got[0] in ("2.gcode", "2a.gcode") + + +def test_case_only_differences_get_a_stable_order(): + shim, _, _ = build(files=[entry("Cube.gcode"), entry("cube.gcode")]) + first = [n for _, n in names(call(shim, order="name"))] + second = [n for _, n in names(call(shim, order="name"))] + assert first == second and set(first) == {"Cube.gcode", "cube.gcode"} + + +def test_desc_reverses_it(): + shim, _, _ = build(files=[entry("a.gcode"), entry("b.gcode")]) + assert [n for _, n in names(call(shim, order="name desc"))] == [ + "b.gcode", "a.gcode"] + + +def test_datetime_and_size_sort_on_their_own_fields(): + shim, _, _ = build(files=[entry("old_big.gcode", modified=1.0, size=900), + entry("new_small.gcode", modified=9.0, size=1)]) + assert [n for _, n in names(call(shim, order="datetime desc"))][0] == "new_small.gcode" + assert [n for _, n in names(call(shim, order="size desc"))][0] == "old_big.gcode" + + +def test_an_unknown_order_falls_back_to_name_ascending(): + """nexusp answers 'files', 'type' and outright nonsense identically, and + with a list rather than an error. Pinned so a future edit does not decide + that an unrecognised sort deserves a 400.""" + shim, _, _ = build(files=[entry("b.gcode"), entry("a.gcode")]) + for order in ("files", "type", "bogus"): + assert [n for _, n in names(call(shim, order=order))] == [ + "a.gcode", "b.gcode"], order + + +# -------------------------------------------------------------------------- +# Keyword, paging, count +# -------------------------------------------------------------------------- + +def test_keyword_is_a_case_insensitive_substring(): + shim, _, _ = build(files=[entry("Cube-4m59s.gcode"), entry("Other.gcode")]) + for kw in ("cube", "CUBE", "ube-4"): + assert names(call(shim, keyword=kw)) == [("f", "Cube-4m59s.gcode")], kw + + +def test_keyword_narrows_the_count_not_just_the_page(): + shim, _, _ = build(files=[entry("a.gcode"), entry("b.gcode"), entry("aa.gcode")]) + assert call(shim, keyword="a")["count"] == 2 + + +def test_count_is_the_total_before_paging(): + shim, _, _ = build(files=[entry("%02d.gcode" % i) for i in range(10)]) + result = call(shim, start=0, limit=3) + assert result["count"] == 10 and len(result["items"]) == 3 + + +def test_start_is_a_row_offset(): + shim, _, _ = build(files=[entry("%02d.gcode" % i) for i in range(10)]) + assert names(call(shim, start=3, limit=2)) == [("f", "03.gcode"), ("f", "04.gcode")] + + +def test_start_past_the_end_is_an_empty_page_with_the_count_intact(): + shim, _, _ = build(files=[entry("a.gcode")]) + result = call(shim, start=500, limit=3) + assert result["items"] == [] and result["count"] == 1 + + +def test_limit_zero_returns_nothing(): + shim, _, _ = build(files=[entry("a.gcode")]) + result = call(shim, start=0, limit=0) + assert result["items"] == [] and result["count"] == 1 + + +def test_a_negative_start_does_not_wrap_to_the_end_of_the_listing(): + """Python would slice from the tail and hand back real files as if they + were the first page, which reads as corruption rather than a bad request.""" + shim, _, _ = build(files=[entry("a.gcode"), entry("b.gcode"), entry("c.gcode")]) + assert names(call(shim, start=-2, limit=2)) == [("f", "a.gcode"), ("f", "b.gcode")] + + +# -------------------------------------------------------------------------- +# Arguments +# -------------------------------------------------------------------------- + +@pytest.mark.parametrize("missing", ["start", "limit", "order"]) +def test_start_limit_and_order_are_all_required(missing): + shim, _, _ = build(files=[entry("a.gcode")]) + args = {"path": "gcodes", "start": 0, "limit": 5, "order": "name"} + args.pop(missing) + with pytest.raises(ServerError) as exc: + run(shim._handle_directory_ex(FakeWebRequest(**args))) + assert exc.value.message == "Invalid parameter" + + +def test_a_missing_path_says_so_by_name(): + shim, _, _ = build() + with pytest.raises(ServerError) as exc: + run(shim._handle_directory_ex( + FakeWebRequest(start=0, limit=5, order="name"))) + assert exc.value.message == "No data for argument: path" + + +def test_since_and_before_are_accepted_and_ignored(): + """Deliberate. nexusp ignores them — measured, a window excluding almost + every file left the count unchanged — and honouring them here would make + files disappear from a browser that has always shown them.""" + shim, _, _ = build(files=[entry("old.gcode", modified=1.0), + entry("new.gcode", modified=9_000_000_000.0)]) + result = call(shim, since=8_000_000_000, before=8_000_000_001) + assert result["count"] == 2 + + +def test_an_unknown_root_propagates_the_file_managers_error(): + shim, _, _ = build() + with pytest.raises(ServerError) as exc: + call(shim, path="nosuchroot") + assert exc.value.message == "Invalid root path (nosuchroot)" + + +# -------------------------------------------------------------------------- +# root_info +# -------------------------------------------------------------------------- + +def test_root_info_carries_the_path_moonraker_does_not_supply(): + shim, _, _ = build(files=[entry("a.gcode")]) + info = call(shim, path="gcodes/sub")["root_info"] + assert info == {"name": "gcodes", "permissions": "rw", "path": "gcodes/sub"} + + +# -------------------------------------------------------------------------- +# server.history.count +# -------------------------------------------------------------------------- + +def test_history_count_returns_the_row_count(): + shim, _, _ = build(history=42) + assert run(shim._handle_history_count(FakeWebRequest())) == {"count": 42} + + +def test_history_count_is_scoped_the_way_the_history_list_is(): + """A count that disagrees with the list it sizes is the same + plausible-wrong-answer failure retiring nexusp exists to kill. Moonraker's + `server.history.list` filters on a bare "default" literal today.""" + shim, _, server = build(history=7) + run(shim._handle_history_count(FakeWebRequest())) + sql, params = server.components["history"].history_table.queries[0] + assert "COUNT(*)" in sql and "instance_id = ?" in sql and params == ("default",) + + +def test_history_count_follows_the_history_component_if_it_grows_an_instance_id(): + """Derived, not hardcoded, so the docstring's claim stays true when + Moonraker stops using a literal.""" + shim, _, server = build(history=7, instance_id="printer-2") + run(shim._handle_history_count(FakeWebRequest())) + _, params = server.components["history"].history_table.queries[0] + assert params == ("printer-2",) + + +def test_history_count_ignores_any_arguments_sent(): + shim, _, _ = build(history=3) + result = run(shim._handle_history_count( + FakeWebRequest(limit=10, order="datetime desc"))) + assert result == {"count": 3} + + +# -------------------------------------------------------------------------- +# Thumbnails — read off the disk, not out of the metadata +# -------------------------------------------------------------------------- + +def thumbs_dir(tmp_path, *names_): + """A `.thumbs` directory holding the named PNGs, with real byte sizes.""" + d = tmp_path / ".thumbs" + d.mkdir() + for i, name in enumerate(names_): + (d / name).write_bytes(b"x" * (10 + i)) + return str(tmp_path) + + +def test_on_disk_thumbnails_are_added_to_the_metadatas(tmp_path): + """The 66-blank-files bug. Moonraker parses the slicer's sizes out of the + gcode; nexusp ALSO rendered 48/195 into .thumbs and listed them. Reporting + only the parsed set is what left most of the browser blank.""" + root = thumbs_dir(tmp_path, "Cube-48x48.png", "Cube-195x195.png") + shim, _, _ = build(disk_root=root, files=[ + dict(entry("Cube.gcode"), thumbnails=[ + {"width": 320, "height": 320, "size": 9, + "relative_path": ".thumbs/Cube-320x320.png"}])]) + got = call(shim)["items"][0]["thumbnails"] + assert [(t["width"], t["height"]) for t in got] == [(48, 48), (195, 195), (320, 320)] + + +def test_the_metadata_entry_wins_on_a_duplicate_size(tmp_path): + """Moonraker read its entry out of the gcode, which is better provenance + than a filename — so a same-size PNG on disk must not displace it.""" + root = thumbs_dir(tmp_path, "Cube-320x320.png") + original = {"width": 320, "height": 320, "size": 4242, + "relative_path": ".thumbs/from-metadata.png"} + shim, _, _ = build(disk_root=root, generate_thumbnails=False, + files=[dict(entry("Cube.gcode"), thumbnails=[original])]) + got = call(shim)["items"][0]["thumbnails"] + assert got == [original] + + +def test_a_file_with_no_metadata_thumbnails_still_gets_the_disk_ones(tmp_path): + root = thumbs_dir(tmp_path, "Cube-195x195.png") + shim, _, _ = build(disk_root=root, files=[entry("Cube.gcode")]) + got = call(shim)["items"][0]["thumbnails"] + assert [(t["width"], t["height"]) for t in got] == [(195, 195)] + + +def test_thumbnails_are_matched_to_their_own_file(tmp_path): + """Stems full of hyphens and digits are the norm here, so the size suffix + is parsed from the END and the rest must match the gcode stem exactly.""" + root = thumbs_dir(tmp_path, "Cube-4m59s-0.2w-0.1h-195x195.png", + "Other-195x195.png") + shim, _, _ = build(disk_root=root, files=[entry("Cube-4m59s-0.2w-0.1h.gcode")]) + got = call(shim)["items"][0]["thumbnails"] + assert [t["relative_path"] for t in got] == [".thumbs/Cube-4m59s-0.2w-0.1h-195x195.png"] + + +def test_unparseable_names_in_thumbs_are_ignored(tmp_path): + root = thumbs_dir(tmp_path, "Cube-195x195.png", "Cube.png", "README.txt", + "Cube-bigx195.png") + shim, _, _ = build(disk_root=root, files=[entry("Cube.gcode")]) + got = call(shim)["items"][0]["thumbnails"] + assert [t["relative_path"] for t in got] == [".thumbs/Cube-195x195.png"] + + +def test_the_reported_size_is_the_real_byte_size(tmp_path): + root = thumbs_dir(tmp_path, "Cube-195x195.png") + shim, _, _ = build(disk_root=root, files=[entry("Cube.gcode")]) + assert call(shim)["items"][0]["thumbnails"][0]["size"] == 10 + + +def test_a_directory_with_no_thumbs_is_not_an_error(tmp_path): + """Every root except gcodes. The scan must degrade to "none", not raise.""" + shim, _, _ = build(disk_root=str(tmp_path), files=[entry("Cube.gcode")]) + assert call(shim)["items"][0].get("thumbnails") in (None, []) + + +def test_only_the_page_is_decorated(tmp_path): + """Thumbnail work happens AFTER the slice. Sorting reads filename/modified/ + size and never thumbnails, so nothing above needs it — and doing it first + meant a 91-file directory did the work for all 91 to serve 20 rows, on the + event loop that also serves klippy and Fluidd, during a print.""" + root = thumbs_dir(tmp_path, "a-195x195.png", "b-195x195.png") + shim, _, _ = build(disk_root=root, + files=[entry("a.gcode"), entry("b.gcode")]) + result = call(shim, start=0, limit=1) + assert result["count"] == 2 + assert result["items"][0]["filename"] == "a.gcode" + assert result["items"][0]["thumbnails"] + + +def test_an_empty_page_does_no_thumbnail_work(tmp_path): + root = thumbs_dir(tmp_path, "a-195x195.png") + shim, _, _ = build(disk_root=root, files=[entry("a.gcode")]) + assert call(shim, start=0, limit=0)["items"] == [] + + +# -------------------------------------------------------------------------- +# Thumbnail generation — the sizes nexusp used to render and Moonraker does not +# -------------------------------------------------------------------------- + +class FakeImage: + """Enough of PIL for this component, and no more. + + Pillow is in neither the workstation nor the venv this repo ships, so the + generation path would otherwise be untestable — and it is the part that + writes to the printer's flash, which is exactly the part worth pinning. + `resize` records its arguments and `save` writes a real file, so assertions + can be about intent (which sizes, from which source) rather than pixels. + """ + + calls = [] + + def __init__(self, path, size): + self.path = path + self.size = size + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def convert(self, mode): + return self + + def resize(self, size, resample=None): + FakeImage.calls.append((self.path, size)) + return FakeImage(self.path, size) + + def save(self, dest): + with open(dest, "wb") as fh: + fh.write(b"png" * self.size[0]) + + +@pytest.fixture +def fake_pil(monkeypatch): + FakeImage.calls = [] + module = types.ModuleType("PIL") + image_mod = types.ModuleType("PIL.Image") + + def _open(path): + # Dimensions come from the filename, which is where they come from on + # the box too — the component never asks PIL for them. + stem = os.path.basename(path).rsplit(".", 1)[0] + w, h = stem.rsplit("-", 1)[1].split("x") + return FakeImage(path, (int(w), int(h))) + + image_mod.open = _open + image_mod.LANCZOS = "LANCZOS" + module.Image = image_mod + monkeypatch.setitem(sys.modules, "PIL", module) + monkeypatch.setitem(sys.modules, "PIL.Image", image_mod) + return FakeImage + + +def test_it_renders_the_two_sizes_moonraker_never_makes(tmp_path, fake_pil): + root = thumbs_dir(tmp_path, "Cube-300x300.png") + shim, _, _ = build(disk_root=root, files=[entry("Cube.gcode")]) + got = call(shim)["items"][0]["thumbnails"] + assert [(t["width"], t["height"]) for t in got] == [(48, 48), (195, 195), (300, 300)] + assert os.path.exists(os.path.join(root, ".thumbs", "Cube-195x195.png")) + + +def test_it_scales_from_the_largest_source_available(tmp_path, fake_pil): + root = thumbs_dir(tmp_path, "Cube-32x32.png", "Cube-300x300.png", + "Cube-96x96.png") + shim, _, _ = build(disk_root=root, files=[entry("Cube.gcode")]) + call(shim) + sources = {os.path.basename(path) for path, _ in fake_pil.calls} + assert sources == {"Cube-300x300.png"} + + +def test_it_never_upscales(tmp_path, fake_pil): + """A 32x32 blown up to a 195x195 tile looks broken, which is worse than + looking absent. 48x48 is still skipped — 32 is smaller than that too.""" + root = thumbs_dir(tmp_path, "Cube-32x32.png") + shim, _, _ = build(disk_root=root, files=[entry("Cube.gcode")]) + got = call(shim)["items"][0]["thumbnails"] + assert [(t["width"], t["height"]) for t in got] == [(32, 32)] + assert fake_pil.calls == [] + + +def test_a_size_already_on_disk_is_not_re_rendered(tmp_path, fake_pil): + """Once per file. This runs on every directory listing, and the write goes + to flash.""" + root = thumbs_dir(tmp_path, "Cube-300x300.png", "Cube-48x48.png", + "Cube-195x195.png") + shim, _, _ = build(disk_root=root, files=[entry("Cube.gcode")]) + call(shim) + assert fake_pil.calls == [] + + +def test_generation_can_be_turned_off(tmp_path, fake_pil): + root = thumbs_dir(tmp_path, "Cube-300x300.png") + shim, _, _ = build(disk_root=root, files=[entry("Cube.gcode")], + generate_thumbnails=False) + got = call(shim)["items"][0]["thumbnails"] + assert [(t["width"], t["height"]) for t in got] == [(300, 300)] + assert fake_pil.calls == [] + + +def test_a_file_with_no_thumbnails_at_all_is_left_alone(tmp_path, fake_pil): + """Nothing to scale FROM. Generating would mean inventing an image — which + is every freshly uploaded file on a box without Pillow, because Moonraker's + own metadata parser needs Pillow to produce the embedded set.""" + root = thumbs_dir(tmp_path) + shim, _, _ = build(disk_root=root, files=[entry("Cube.gcode")]) + assert call(shim)["items"][0].get("thumbnails") in (None, []) + assert fake_pil.calls == [] + + +def test_no_thumbs_directory_is_created_for_a_file_with_nothing_to_scale( + tmp_path, fake_pil): + """A directory of thumbnail-less gcode must not grow an empty `.thumbs` + just because it was browsed.""" + shim, _, _ = build(disk_root=str(tmp_path), files=[entry("Cube.gcode")]) + call(shim) + assert not os.path.exists(os.path.join(str(tmp_path), ".thumbs")) + + +def test_a_render_failure_does_not_break_the_listing(tmp_path, fake_pil, monkeypatch): + """One unreadable PNG must not take the whole file browser down with it.""" + def boom(path): + raise OSError("cannot identify image file") + monkeypatch.setattr(sys.modules["PIL.Image"], "open", boom) + root = thumbs_dir(tmp_path, "Cube-300x300.png") + shim, _, _ = build(disk_root=root, files=[entry("Cube.gcode")]) + got = call(shim)["items"][0]["thumbnails"] + assert [(t["width"], t["height"]) for t in got] == [(300, 300)] + + +def test_a_failed_render_is_not_retried_forever(tmp_path, fake_pil, monkeypatch): + """Otherwise a broken PNG means a traceback on every listing, for as long as + the file exists. A Moonraker restart clears the memo, so a repaired + filesystem heals on its own.""" + attempts = [] + + def boom(path): + attempts.append(path) + raise OSError("cannot identify image file") + monkeypatch.setattr(sys.modules["PIL.Image"], "open", boom) + root = thumbs_dir(tmp_path, "Cube-300x300.png") + shim, _, _ = build(disk_root=root, files=[entry("Cube.gcode")]) + call(shim) + first = len(attempts) + call(shim) + call(shim) + assert first == 2 and len(attempts) == 2 + + +# -------------------------------------------------------------------------- +# Pillow is NOT in the venv this repo ships +# -------------------------------------------------------------------------- + +def test_a_missing_pillow_turns_generation_off_rather_than_failing(tmp_path): + """`tar -tzf files/moonraker/moonraker.tar.gz | grep -ci "PIL/\\|pillow"` + returns 0, so this is the state a clean helper install is in. A directory + listing must never fail because an optional renderer is absent.""" + root = thumbs_dir(tmp_path, "Cube-300x300.png") + shim, _, _ = build(disk_root=root, files=[entry("Cube.gcode")]) + assert shim.generate_thumbs is False + got = call(shim)["items"][0]["thumbnails"] + assert [(t["width"], t["height"]) for t in got] == [(300, 300)] + + +def test_a_missing_pillow_still_lists_every_thumbnail_on_disk(tmp_path): + """The disk-union listing is most of the fix and does not need Pillow at + all — it is what put the 66 blank files back.""" + root = thumbs_dir(tmp_path, "Cube-48x48.png", "Cube-195x195.png", + "Cube-300x300.png") + shim, _, _ = build(disk_root=root, files=[entry("Cube.gcode")]) + got = call(shim)["items"][0]["thumbnails"] + assert [(t["width"], t["height"]) for t in got] == [ + (48, 48), (195, 195), (300, 300)] + + +# -------------------------------------------------------------------------- +# `.thumbs` is kept out of file_manager's inotify watch +# -------------------------------------------------------------------------- + +def test_existing_thumb_dirs_are_reserved_at_load(tmp_path, fake_pil): + """Writing a PNG into a watched directory fires `notify_filelist_changed`, + and file_manager has no dot-directory filter — so generating thumbnails + inside a listing would broadcast a phantom `create_file` for every PNG to + Fluidd and to the screen. Reserving has to happen before file_manager's + initial scan, which runs in its `component_init`, after every component is + constructed.""" + root = thumbs_dir(tmp_path, "Cube-195x195.png") + sub = tmp_path / "sub" / ".thumbs" + sub.mkdir(parents=True) + _, fm, _ = build(disk_root=root) + reserved = {path for path, _ in fm.reserved.values()} + assert reserved == {os.path.join(root, ".thumbs"), str(sub)} + + +def test_reserved_thumb_dirs_stay_readable(tmp_path, fake_pil): + """Read access is left on, or the thumbnails stop being served over HTTP + and every tile goes blank — the exact failure this is meant to fix.""" + root = thumbs_dir(tmp_path, "Cube-195x195.png") + _, fm, _ = build(disk_root=root) + assert all(read_access for _, read_access in fm.reserved.values()) + + +def test_a_new_thumb_dir_is_reserved_before_it_is_created(tmp_path, fake_pil): + """The directory-create inotify event checks the reserved list, so the + reservation only works if it lands first. Here `.thumbs` does not exist at + load — the source is a slicer-embedded thumbnail — so the component has to + reserve it on the way to creating it.""" + root = str(tmp_path) + shim, fm, _ = build(disk_root=root, files=[ + dict(entry("Cube.gcode"), thumbnails=[ + {"width": 300, "height": 300, "size": 9, + "relative_path": ".thumbs/Cube-300x300.png"}])]) + assert fm.reserved == {} + call(shim) + assert os.path.join(root, ".thumbs") in {p for p, _ in fm.reserved.values()} + assert os.path.isdir(os.path.join(root, ".thumbs")) + + +def test_nothing_is_reserved_when_generation_is_off(tmp_path, fake_pil): + """No writes, no phantom notifications, no reason to hide `.thumbs` from + Fluidd's file list.""" + root = thumbs_dir(tmp_path, "Cube-195x195.png") + _, fm, _ = build(disk_root=root, generate_thumbnails=False) + assert fm.reserved == {} + + +def test_nothing_is_reserved_when_pillow_is_missing(tmp_path): + """Same reasoning: without Pillow the component never writes a PNG, so + there is nothing to keep out of the watch.""" + root = thumbs_dir(tmp_path, "Cube-195x195.png") + _, fm, _ = build(disk_root=root) + assert fm.reserved == {} + + +# -------------------------------------------------------------------------- +# Sort parsing — the reason the screen's sort buttons did nothing +# -------------------------------------------------------------------------- + +def test_the_sort_field_is_matched_case_insensitively(): + """The bug. The first version compared the field case-sensitively, so a + screen sending "Datetime" fell through to the name-ascending default — + silently, because falling back rather than erroring is nexusp's behaviour.""" + shim, _, _ = build(files=[entry("a.gcode", modified=1.0), + entry("b.gcode", modified=9.0)]) + for order in ("DATETIME DESC", "Datetime Desc", "datetime desc"): + assert [n for _, n in names(call(shim, order=order))] == [ + "b.gcode", "a.gcode"], order + + +@pytest.mark.parametrize("order", ["datetime,desc", "datetime;desc", + "datetime|desc", "datetime desc"]) +def test_the_direction_survives_any_plausible_separator(order): + """The screen's exact encoding was never measured — nexusp was only ever + asked in the form the binary's strings suggested. Splitting on whitespace + alone is a guess that fails silently, so this splits on all of them.""" + shim, _, _ = build(files=[entry("a.gcode", modified=1.0), + entry("b.gcode", modified=9.0)]) + assert [n for _, n in names(call(shim, order=order))] == ["b.gcode", "a.gcode"] + + +@pytest.mark.parametrize("alias", ["date desc", "time desc", "modified desc", + "mtime desc"]) +def test_time_aliases_all_sort_by_modified(alias): + shim, _, _ = build(files=[entry("a.gcode", modified=1.0), + entry("b.gcode", modified=9.0)]) + assert [n for _, n in names(call(shim, order=alias))][0] == "b.gcode" + + +@pytest.mark.parametrize("alias", ["size desc", "filesize desc", "bytes desc"]) +def test_size_aliases_all_sort_by_size(alias): + shim, _, _ = build(files=[entry("small.gcode", size=1), + entry("big.gcode", size=999)]) + assert [n for _, n in names(call(shim, order=alias))][0] == "big.gcode" + + +def test_the_order_string_the_screen_actually_sends(): + """Measured from the box on 2026-08-02 with `log_requests: True`. The comma + is the whole reason the sort buttons did nothing: split on whitespace alone, + `name,asc,folder` is one token that matches no field.""" + shim, _, _ = build(files=[entry("a.gcode", modified=1.0, size=999), + entry("b.gcode", modified=9.0, size=1)]) + + def order_of(o): + return [n for _, n in names(call(shim, order=o))] + + assert order_of("name,asc,folder") == ["a.gcode", "b.gcode"] + assert order_of("datetime,desc,folder") == ["b.gcode", "a.gcode"] + assert order_of("datetime,asc,folder") == ["a.gcode", "b.gcode"] + assert order_of("size,desc,folder") == ["a.gcode", "b.gcode"] + assert order_of("size,asc,folder") == ["b.gcode", "a.gcode"] + + +def test_the_full_argument_set_the_screen_sends_is_accepted(): + """It sends `root` beside `path`, and zeroes for since/before plus an empty + keyword on every call — none of which may be mistaken for a filter.""" + shim, _, _ = build(files=[entry("a.gcode"), entry("b.gcode")]) + result = call(shim, root="gcodes", path="gcodes", limit=5, start=0, + since=0, before=0, order="name,asc,folder", keyword="") + assert result["count"] == 2 and len(result["items"]) == 2 diff --git a/files/moonraker/creality-compat/test_merge_job_history.py b/files/moonraker/creality-compat/test_merge_job_history.py new file mode 100644 index 0000000..b414747 --- /dev/null +++ b/files/moonraker/creality-compat/test_merge_job_history.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python3 +"""Offline checks for merge_job_history.py — no printer, no daemon, no network. + +Standalone, no conftest and no fixtures beyond this file: + + python3 -m pytest -q test_merge_job_history.py + +WHY THIS FILE EXISTS +-------------------- +merge_job_history.py is the only artifact in the nexusp retirement that mutates +irreplaceable data. On the reference unit 22 of 42 print records existed in +exactly one place — nexusp's database — and the merge is the single event that +decides whether they survive. It runs once per direction, on a machine where a +mistake is discovered weeks later by noticing an absence. So the safety +properties get pinned down here, where they can be exercised as often as we +like, rather than on the printer. + +Four of them are load-bearing and none is observable at merge time: + + 1. THE BACKUP IS RESTORABLE. `.bak-merge-` is the entire rollback story, + and until this file existed nothing had ever opened one. test_backup_ + restores_the_pre_merge_state does the whole round trip. + 2. THE DEDUPE WINDOW HAS THE RIGHT SHAPE. Too narrow and shared prints double + up (visibly, harmlessly); too wide and a distinct print is silently + swallowed by its neighbour and is simply gone. The boundary tests pin 120 s + as inclusive and pin what falls either side of it. + 3. A DROPPED ROW IS REPORTED AS A DROP. in_progress rows are deliberately not + copied. An earlier draft counted them as duplicates, which reads on the dry + run as "already there" — the one lie a dry run must not tell. + 4. BOTH DIRECTIONS WORK. Restore hands the touchscreen back a database frozen + at the retirement date unless the prints made while retired are merged into + it first, and that loss is silent. + +THE SCHEMAS ARE THE REAL ONES, copied verbatim from the reference unit +(2026-08-02) rather than written from memory. Both matter: `job_totals`' +composite PRIMARY KEY is what makes the merge's `insert or replace` a replace +instead of a duplicate, and `job_id INTEGER PRIMARY KEY ASC` is what makes +inserted ids continue after the target's. + +⚠ Add cases as `test_*` functions. A file named test_*.py whose assertions live +in a hand-rolled runner gets collected and runs nothing while reporting green. +""" +import glob +import os +import shutil +import sqlite3 +import sys + +import pytest + +REPO = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, REPO) + +import merge_job_history as mjh # noqa: E402 + +# Verbatim from the reference unit. moonraker declares metadata/auxiliary_data +# as `pyjson` and nexusp as TEXT; sqlite type names are advisory and the script +# connects without detect_types, so one DDL serves both here. +JOB_HISTORY_DDL = """ +CREATE TABLE job_history ( + job_id INTEGER PRIMARY KEY ASC, + user TEXT NOT NULL, + filename TEXT, + status TEXT NOT NULL, + start_time REAL NOT NULL, + end_time REAL, + print_duration REAL NOT NULL, + total_duration REAL NOT NULL, + filament_used REAL NOT NULL, + metadata pyjson, + auxiliary_data pyjson NOT NULL, + instance_id TEXT NOT NULL +) +""" + +JOB_TOTALS_DDL = """ +CREATE TABLE job_totals ( + provider TEXT NOT NULL, + field TEXT NOT NULL, + maximum REAL, + total REAL, + instance_id TEXT NOT NULL, + PRIMARY KEY (provider, field, instance_id) +) +""" + +T0 = 1_750_000_000.0 # an arbitrary fixed epoch; nothing here depends on "now" + + +def job(start, filename="a.gcode", status="completed", job_id=None, user="_TRUSTED", + end=None, print_duration=100.0, total_duration=120.0, filament_used=1.0, + instance_id="default"): + """One job_history row as the dict the helpers below insert.""" + return { + "job_id": job_id, + "user": user, + "filename": filename, + "status": status, + "start_time": start, + "end_time": start + total_duration if end is None else end, + "print_duration": print_duration, + "total_duration": total_duration, + "filament_used": filament_used, + "metadata": "{}", + "auxiliary_data": "[]", + "instance_id": instance_id, + } + + +def total(field, maximum=0.0, tot=0.0, provider="history", instance_id="default"): + return {"provider": provider, "field": field, "maximum": maximum, + "total": tot, "instance_id": instance_id} + + +def make_db(path, jobs=(), totals=()): + con = sqlite3.connect(str(path)) + con.execute(JOB_HISTORY_DDL) + con.execute(JOB_TOTALS_DDL) + for j in jobs: + cols = [c for c in j if not (c == "job_id" and j[c] is None)] + con.execute("insert into job_history (%s) values (%s)" + % (",".join(cols), ",".join("?" * len(cols))), + tuple(j[c] for c in cols)) + for t in totals: + con.execute("insert into job_totals (provider, field, maximum, total, " + "instance_id) values (?,?,?,?,?)", + (t["provider"], t["field"], t["maximum"], t["total"], + t["instance_id"])) + con.commit() + con.close() + return str(path) + + +def rows_of(path, table="job_history", order="start_time"): + con = sqlite3.connect(path) + con.row_factory = sqlite3.Row + out = [dict(r) for r in con.execute("select * from %s order by %s" % (table, order))] + con.close() + return out + + +def run_main(monkeypatch, capsys, into, source, *flags): + """Drive main() with argv, returning its stdout. Raises SystemExit on refusal.""" + monkeypatch.setattr(sys, "argv", + ["merge_job_history.py", "--into", into, + "--source", source] + list(flags)) + mjh.main() + return capsys.readouterr().out + + +def run_argv(monkeypatch, capsys, *flags): + """Drive main() with no path overrides, so the direction defaults apply.""" + monkeypatch.setattr(sys, "argv", ["merge_job_history.py"] + list(flags)) + mjh.main() + return capsys.readouterr().out + + +# -------------------------------------------------------------------------- +# The dedupe window +# -------------------------------------------------------------------------- + +def test_same_file_inside_the_window_is_a_duplicate(): + target = [job(T0)] + new, dupes, skipped = mjh.classify([job(T0 + 119)], target) + assert (len(new), len(dupes), len(skipped)) == (0, 1, 0) + + +def test_the_window_boundary_is_inclusive(): + """Exactly TOLERANCE_S counts as the same print. + + Which side of the boundary the equal case falls on is arbitrary; that it is + PINNED is not. Widening the window later without noticing this test is how a + distinct print gets swallowed. + """ + target = [job(T0)] + new, dupes, _ = mjh.classify([job(T0 + mjh.TOLERANCE_S)], target) + assert (len(new), len(dupes)) == (0, 1) + + +def test_same_file_outside_the_window_is_a_distinct_print(): + target = [job(T0)] + new, dupes, _ = mjh.classify([job(T0 + 121)], target) + assert (len(new), len(dupes)) == (1, 0) + + +def test_a_failed_print_and_its_retry_both_survive(): + """The realistic worst case for a time-based dedupe. + + Same filename, minutes apart, first one cancelled. Measured on the reference + unit the closest such pair is >600 s; this asserts the pair stays two rows. + """ + target = [job(T0, status="klippy_shutdown")] + new, dupes, _ = mjh.classify([job(T0, status="klippy_shutdown"), + job(T0 + 610)], target) + assert (len(new), len(dupes)) == (1, 1) + + +def test_different_files_at_the_same_instant_are_distinct(): + target = [job(T0, filename="a.gcode")] + new, dupes, _ = mjh.classify([job(T0, filename="b.gcode")], target) + assert (len(new), len(dupes)) == (1, 0) + + +def test_empty_target_takes_everything(): + new, dupes, skipped = mjh.classify([job(T0), job(T0 + 9999)], []) + assert (len(new), len(dupes), len(skipped)) == (2, 0, 0) + + +def test_empty_source_is_a_no_op(): + assert mjh.classify([], [job(T0)]) == ([], [], []) + + +# -------------------------------------------------------------------------- +# in_progress: dropped, and REPORTED as dropped +# -------------------------------------------------------------------------- + +def test_in_progress_is_skipped_not_counted_as_a_duplicate(): + target = [job(T0)] + new, dupes, skipped = mjh.classify( + [job(T0 + 1, status="in_progress", end=None)], target) + assert (len(new), len(dupes), len(skipped)) == (0, 0, 1) + + +def test_in_progress_without_a_twin_is_still_skipped(): + """No twin in the target, so it is a genuine loss — and must still not be + copied: a row with end_time NULL is a job that never completes.""" + new, dupes, skipped = mjh.classify( + [job(T0, status="in_progress", end=None)], []) + assert (len(new), len(dupes), len(skipped)) == (0, 0, 1) + + +def test_the_dry_run_lists_every_skipped_row(monkeypatch, capsys, tmp_path): + into = make_db(tmp_path / "into.db", [job(T0)]) + source = make_db(tmp_path / "src.db", + [job(T0 + 1, filename="live.gcode", status="in_progress", + end=None)]) + out = run_main(monkeypatch, capsys, into, source) + assert "in_progress (NOT copied): 1" in out + assert "live.gcode" in out + + +# -------------------------------------------------------------------------- +# job_totals +# -------------------------------------------------------------------------- + +def test_totals_take_the_max_of_each_column_independently(): + """Row A holds the bigger lifetime sum, row B the bigger single-job record; + taking whole rows would throw one of them away.""" + a = [total("longest_print", maximum=100.0, tot=5000.0)] + b = [total("longest_print", maximum=900.0, tot=10.0)] + merged = mjh.merge_totals(a, b) + assert merged[("history", "longest_print", "default")] == { + "maximum": 900.0, "total": 5000.0} + + +def test_a_null_column_never_wins(): + a = [total("total_jobs", maximum=None, tot=184.0)] + b = [total("total_jobs", maximum=7.0, tot=None)] + merged = mjh.merge_totals(a, b) + assert merged[("history", "total_jobs", "default")] == { + "maximum": 7.0, "total": 184.0} + + +def test_instances_do_not_merge_into_each_other(): + a = [total("total_jobs", tot=184.0, instance_id="default")] + b = [total("total_jobs", tot=17.0, instance_id="other")] + merged = mjh.merge_totals(a, b) + assert merged[("history", "total_jobs", "default")]["total"] == 184.0 + assert merged[("history", "total_jobs", "other")]["total"] == 17.0 + + +def test_fields_do_not_merge_into_each_other(): + merged = mjh.merge_totals([total("total_jobs", tot=184.0)], + [total("total_time", tot=3.0)]) + assert len(merged) == 2 + + +def test_totals_land_in_the_database_as_a_replace(monkeypatch, capsys, tmp_path): + """job_totals' composite PRIMARY KEY is what makes `insert or replace` a + replace. If that key were ever lost the row count would grow instead.""" + into = make_db(tmp_path / "into.db", [], [total("total_jobs", tot=17.0)]) + source = make_db(tmp_path / "src.db", [], [total("total_jobs", tot=184.0)]) + run_main(monkeypatch, capsys, into, source, "--apply") + after = rows_of(into, "job_totals", "field") + assert len(after) == 1 + assert after[0]["total"] == 184.0 + + +# -------------------------------------------------------------------------- +# The guards +# -------------------------------------------------------------------------- + +def test_a_live_daemon_blocks_and_force_overrides(monkeypatch, capsys, tmp_path): + monkeypatch.setattr(mjh, "moonraker_running", lambda: "python moonraker.py") + into = make_db(tmp_path / "into.db", [job(T0)]) + source = make_db(tmp_path / "src.db", [job(T0 + 9999)]) + with pytest.raises(SystemExit): + run_main(monkeypatch, capsys, into, source, "--apply") + assert len(rows_of(into)) == 1 + assert glob.glob(into + ".bak-merge-*") == [] + run_main(monkeypatch, capsys, into, source, "--apply", "--force") + assert len(rows_of(into)) == 2 + + +def test_no_proc_reads_as_no_daemon(monkeypatch): + """The workstation rehearsal path: /proc does not exist on a Mac, and the + guard must degrade to "nothing running here" rather than crash.""" + monkeypatch.setattr(os.path, "isdir", lambda p: False) + assert mjh.moonraker_running() is None + + +def test_a_missing_database_exits_before_touching_anything(monkeypatch, capsys, + tmp_path): + into = make_db(tmp_path / "into.db", [job(T0)]) + with pytest.raises(SystemExit): + run_main(monkeypatch, capsys, into, str(tmp_path / "nope.db"), "--apply") + assert glob.glob(into + ".bak-merge-*") == [] + + +# -------------------------------------------------------------------------- +# Direction +# -------------------------------------------------------------------------- + +def test_the_default_direction_is_nexusp_into_moonraker(): + assert mjh.DIRECTIONS["to-moonraker"] == (mjh.NEXUSP_DB, mjh.MOONRAKER_DB) + + +def test_the_restore_direction_is_the_exact_reverse(): + """Restore hands the screen back nexusp-sql.db, frozen at the retirement + date, unless everything printed while retired goes back into it first — and + that loss is silent, with nothing to connect it to the restore.""" + assert mjh.DIRECTIONS["to-nexusp"] == (mjh.MOONRAKER_DB, mjh.NEXUSP_DB) + + +def test_direction_picks_the_databases(monkeypatch, capsys, tmp_path): + moonraker = make_db(tmp_path / "moonraker-sql.db", + [job(T0, filename="while_retired.gcode")]) + nexusp = make_db(tmp_path / "nexusp-sql.db", + [job(T0 - 86400, filename="before_retirement.gcode")]) + monkeypatch.setattr(mjh, "DIRECTIONS", { + "to-moonraker": (nexusp, moonraker), + "to-nexusp": (moonraker, nexusp), + }) + run_argv(monkeypatch, capsys, "--direction", "to-nexusp", "--apply") + assert [r["filename"] for r in rows_of(nexusp)] == [ + "before_retirement.gcode", "while_retired.gcode"] + # The forward database is untouched by a backward merge. + assert [r["filename"] for r in rows_of(moonraker)] == ["while_retired.gcode"] + + +def test_the_direction_is_reported_on_the_dry_run(monkeypatch, capsys, tmp_path): + """The one thing a user can check before letting it write.""" + into = make_db(tmp_path / "into.db", [job(T0)]) + source = make_db(tmp_path / "src.db", [job(T0 + 9999)]) + out = run_main(monkeypatch, capsys, into, source, "--direction", "to-nexusp") + assert "direction: to-nexusp" in out + + +def test_a_round_trip_does_not_duplicate_the_shared_prints(monkeypatch, capsys, + tmp_path): + """Retire, print, restore. The prints made while retired must arrive in + nexusp's database exactly once, and the ones that were merged forward at + retirement must not come back as copies.""" + moonraker = make_db(tmp_path / "moonraker-sql.db", + [job(T0, filename="shared.gcode")]) + nexusp = make_db(tmp_path / "nexusp-sql.db", + [job(T0 + 0.7, filename="shared.gcode"), + job(T0 - 86400, filename="ancient.gcode")]) + run_main(monkeypatch, capsys, moonraker, nexusp, "--apply") + assert sorted(r["filename"] for r in rows_of(moonraker)) == [ + "ancient.gcode", "shared.gcode"] + + # ... a print happens while nexusp is retired ... + con = sqlite3.connect(moonraker) + j = job(T0 + 500000, filename="while_retired.gcode") + cols = [c for c in j if c != "job_id"] + con.execute("insert into job_history (%s) values (%s)" + % (",".join(cols), ",".join("?" * len(cols))), + tuple(j[c] for c in cols)) + con.commit() + con.close() + + run_main(monkeypatch, capsys, nexusp, moonraker, "--apply") + assert sorted(r["filename"] for r in rows_of(nexusp)) == [ + "ancient.gcode", "shared.gcode", "while_retired.gcode"] + + +# -------------------------------------------------------------------------- +# End to end +# -------------------------------------------------------------------------- + +def test_the_dry_run_writes_nothing(monkeypatch, capsys, tmp_path): + into = make_db(tmp_path / "into.db", [job(T0)]) + source = make_db(tmp_path / "src.db", [job(T0 + 9999), job(T0 + 19999)]) + before = rows_of(into) + out = run_main(monkeypatch, capsys, into, source) + assert "to insert: 2" in out + assert rows_of(into) == before + assert glob.glob(into + ".bak-merge-*") == [] + + +def test_apply_inserts_the_missing_prints(monkeypatch, capsys, tmp_path): + into = make_db(tmp_path / "into.db", [job(T0, filename="shared.gcode")]) + source = make_db(tmp_path / "src.db", + [job(T0 + 0.7, filename="shared.gcode"), + job(T0 - 86400, filename="older.gcode")]) + run_main(monkeypatch, capsys, into, source, "--apply") + got = [r["filename"] for r in rows_of(into)] + assert got == ["older.gcode", "shared.gcode"] + + +def test_backup_restores_the_pre_merge_state(monkeypatch, capsys, tmp_path): + """THE ROLLBACK. Nothing else in this change verifies that the file the + script writes before its first insert can actually be put back.""" + into = make_db(tmp_path / "into.db", [job(T0, filename="shared.gcode")], + [total("total_jobs", tot=17.0)]) + source = make_db(tmp_path / "src.db", [job(T0 - 86400, filename="older.gcode")], + [total("total_jobs", tot=184.0)]) + before_jobs, before_totals = rows_of(into), rows_of(into, "job_totals", "field") + + run_main(monkeypatch, capsys, into, source, "--apply") + assert len(rows_of(into)) == 2 + assert rows_of(into, "job_totals", "field")[0]["total"] == 184.0 + + backups = glob.glob(into + ".bak-merge-*") + assert len(backups) == 1 + shutil.copy2(backups[0], into) + + assert rows_of(into) == before_jobs + assert rows_of(into, "job_totals", "field") == before_totals + + +def test_a_second_apply_inserts_nothing(monkeypatch, capsys, tmp_path): + into = make_db(tmp_path / "into.db", [job(T0)]) + source = make_db(tmp_path / "src.db", [job(T0 - 86400, filename="older.gcode")]) + run_main(monkeypatch, capsys, into, source, "--apply") + first = rows_of(into) + out = run_main(monkeypatch, capsys, into, source, "--apply") + assert "to insert: 0" in out + assert rows_of(into) == first + + +def test_existing_job_ids_are_never_rewritten(monkeypatch, capsys, tmp_path): + """Renumbering would rewrite the identity of rows a client may already + hold — a Fluidd tab left open across the merge would then delete by an id + that names a different print. Inserted rows continue after the target's. + """ + into = make_db(tmp_path / "into.db", + [job(T0, job_id=41, filename="a.gcode"), + job(T0 + 20000, job_id=42, filename="b.gcode")]) + source = make_db(tmp_path / "src.db", + [job(T0 - 86400, filename="ancient.gcode")]) + run_main(monkeypatch, capsys, into, source, "--apply") + by_name = {r["filename"]: r["job_id"] for r in rows_of(into)} + assert by_name["a.gcode"] == 41 + assert by_name["b.gcode"] == 42 + # The oldest print by time gets the HIGHEST id. That inversion is the + # deliberate trade: ids stay stable, ordering is start_time's job. + assert by_name["ancient.gcode"] == 43 diff --git a/files/moonraker/moonraker.conf b/files/moonraker/moonraker.conf index 38506c8..7bcc124 100644 --- a/files/moonraker/moonraker.conf +++ b/files/moonraker/moonraker.conf @@ -77,6 +77,11 @@ managed_services: klipper #ffmpeg_binary_path: /opt/bin/ffmpeg #snapshoturl: http://localhost:8080/?action=snapshot +# Remove '#' after this line if you retired Creality's nexusp backend (K1C 2025 only). The 'Retire Nexusp Backend' option does this for you +#[creality_compat] +#generate_thumbnails: True +#log_requests: False + # Remove '#' after this line if you use Fluidd #[update_manager fluidd] #type: web diff --git a/files/services/S50nginx b/files/services/S50nginx index 2bd89f9..91e3887 100755 --- a/files/services/S50nginx +++ b/files/services/S50nginx @@ -19,7 +19,11 @@ case "$1" in ;; reload|force-reload) echo "Reloading nginx..." - "$NGINX" -s reload + # NGINX_ARGS, not bare: without -c nginx opens /etc/nginx/nginx.conf, which + # does not exist on this board. The reload then fails and leaves the OLD + # config live while reporting nothing, so a config change looks applied and + # Fluidd 502s until somebody works out why. + "$NGINX" $NGINX_ARGS -s reload ;; restart) "$0" stop diff --git a/scripts/disable_creality_services.sh b/scripts/disable_creality_services.sh index d0f42bc..2438842 100755 --- a/scripts/disable_creality_services.sh +++ b/scripts/disable_creality_services.sh @@ -12,8 +12,10 @@ set -e # # Services this option must never disable, whatever a later edit is tempted to add: # klipper_service - Klipper itself. -# nexusp_service - the touchscreen backend on :7125; the GUI holds an open -# connection to it. +# nexusp_service - the touchscreen backend on :7125. Retiring it is possible, +# but it needs a compatibility shim for two JSON-RPC methods +# the screen calls and a print-history merge, so it belongs +# to Retire Nexusp Backend (retire_nexusp.sh) and not here. # quintusp - HAL for the LCD backlight, chassis LED, camera arbitration # and the power-loss GPIO. Disabling it breaks power-loss # recovery, and it can cancel a running print. @@ -104,9 +106,12 @@ function creality_builtin_camera_fix_installed() { } # 0 = printing or paused, 1 = confirmed idle, 2 = could not be determined. -# Both ports are consulted and any "printing" wins: helper Moonraker moves to 7126 on -# this model (moonraker_nginx.sh), while nexusp answers on 7125 whether or not Moonraker -# is installed. Only an explicitly known idle state counts as idle - anything else falls +# Both ports are consulted and any "printing" wins, because which daemon answers which +# port depends on whether nexusp has been retired: normally helper Moonraker is on 7126 +# (moonraker_nginx.sh) and nexusp on 7125, and after Retire Nexusp Backend the helper's +# Moonraker is on 7125 and nothing is on 7126. Trying both covers either arrangement - +# the unused port simply does not answer. Only an explicitly known idle state counts as +# idle - anything else falls # through to 2 so the caller asks the user rather than assuming the printer is free. # jq is not available on the K1_2025 path, so the state is pulled out with sed/grep, and # the body is trimmed to the print_stats object first so an unrelated "state" key in diff --git a/scripts/menu/K1_2025/customize_menu_K1C_2025.sh b/scripts/menu/K1_2025/customize_menu_K1C_2025.sh index 577637a..424e961 100755 --- a/scripts/menu/K1_2025/customize_menu_K1C_2025.sh +++ b/scripts/menu/K1_2025/customize_menu_K1C_2025.sh @@ -12,6 +12,9 @@ function customize_menu_ui_k1_2025() { menu_option '2' 'Disable' 'Creality Stock Services' menu_option '3' 'Restore' 'Creality Stock Services' hr + menu_option '4' 'Retire' 'Nexusp Backend' + menu_option '5' 'Restore' 'Nexusp Backend' + hr inner_line hr bottom_menu_option 'b' 'Back to [Main Menu]' "${yellow}" @@ -52,6 +55,25 @@ function customize_menu_k1_2025() { else run "restore_creality_services" "customize_menu_ui_k1_2025" fi;; + 4) + if nexusp_absent; then + error_msg "No nexusp service was found on this firmware!" + elif nexusp_retired && ! nexusp_resurrected; then + error_msg "Nexusp Backend is already retired!" + else + # nexusp_resurrected falls through on purpose: a firmware update put + # the service file back and retire_nexusp offers to re-apply the + # rename, which is the only repair for it. + run "retire_nexusp" "customize_menu_ui_k1_2025" + fi;; + 5) + if nexusp_absent; then + error_msg "No nexusp service was found on this firmware!" + elif ! nexusp_retired; then + error_msg "Nexusp Backend is not retired!" + else + run "restore_nexusp" "customize_menu_ui_k1_2025" + fi;; B|b) clear; main_menu; break;; Q|q) diff --git a/scripts/menu/K1_2025/info_menu_K1C_2025.sh b/scripts/menu/K1_2025/info_menu_K1C_2025.sh index 2955beb..f453397 100755 --- a/scripts/menu/K1_2025/info_menu_K1C_2025.sh +++ b/scripts/menu/K1_2025/info_menu_K1C_2025.sh @@ -63,6 +63,34 @@ function check_creality_services_k1_2025() { fi } +# Tri-state. `~` is not "half done" here, it is the state a firmware update +# leaves behind: the service file recreated beside the disabled copy, losing the +# race for port 7125 to Moonraker at every boot. Retire Nexusp Backend offers +# the repair. +function check_nexusp_retired_k1_2025() { + if nexusp_resurrected; then + echo -e "${yellow}~" + elif nexusp_retired; then + echo -e "${green}✓" + else + echo -e "${red}✗" + fi +} + +# The port Moonraker actually listens on, which is the whole point of retiring +# nexusp - and the one thing a user needs to know before pasting any command +# from a Klipper forum at this printer. Plain text, no colour escapes: info_line +# pads on ${#status} and a second escaped field would push the box out of shape. +function check_moonraker_port_k1_2025() { + local port + port="$(nexusp_moonraker_port)" + if [ -z "$port" ]; then + echo "port unknown" + else + echo "port $port" + fi +} + function info_menu_ui_k1_2025() { top_line title '[ INFORMATION MENU ]' "${yellow}" @@ -107,6 +135,7 @@ function info_menu_ui_k1_2025() { subtitle '•CUSTOMIZATION:' info_line "$(check_file_k1_2025 "$FLUIDD_LOGO_FILE")" 'Creality Dynamic Logos for Fluidd' info_line "$(check_creality_services_k1_2025)" 'Creality Stock Services Disabled' + info_line "$(check_nexusp_retired_k1_2025)" "Nexusp Backend Retired (Moonraker on $(check_moonraker_port_k1_2025))" hr inner_line hr diff --git a/scripts/moonraker_nginx.sh b/scripts/moonraker_nginx.sh index be56a74..f66a55c 100755 --- a/scripts/moonraker_nginx.sh +++ b/scripts/moonraker_nginx.sh @@ -58,6 +58,19 @@ function moonraker_3v3_message(){ function configure_moonraker_nginx_k1_2025(){ local nginx_conf + # Moonraker moves to 7126 because Creality's nexusp squats on 7125 - UNLESS + # Retire Nexusp Backend has already turned nexusp off, in which case 7125 is + # ours and the touchscreen is pointed at it. install_moonraker_nginx rm -f's + # moonraker.conf and re-copies the shipped one, so without this branch a + # Moonraker reinstall on a retired box put the port back to 7126 and wiped + # [creality_compat] while nexusp stayed disabled: nothing answered 7125, the + # touchscreen died, and there was no error anywhere to connect it to. + if nexusp_retired && ! nexusp_present; then + echo -e "Info: Nexusp is retired, keeping Moonraker on port 7125..." + nexusp_reapply_retired_config + return + fi + if [ -f "$MOONRAKER_CFG" ]; then echo -e "Info: Setting Moonraker port to 7126..." sed -i 's/^port:[[:space:]]*7125$/port: 7126/' "$MOONRAKER_CFG" diff --git a/scripts/paths.sh b/scripts/paths.sh index 3748b57..50e5821 100755 --- a/scripts/paths.sh +++ b/scripts/paths.sh @@ -40,17 +40,20 @@ function set_paths() { # Moonraker # MOONRAKER_FOLDER="${USR_DATA}/moonraker" + MOONRAKER_ENV_PYTHON="${MOONRAKER_FOLDER}/moonraker-env/bin/python3" MOONRAKER_URL1="${HS_FILES}/moonraker/moonraker.tar.gz" MOONRAKER_URL2="${HS_FILES}/moonraker/moonraker.conf" MOONRAKER_URL3="${HS_FILES}/moonraker/moonraker.asvc" MOONRAKER_SERVICE_URL="${HS_FILES}/services/S56moonraker_service" - + # Nginx # NGINX_FOLDER="${USR_DATA}/nginx" + NGINX_BIN="${NGINX_FOLDER}/sbin/nginx" + NGINX_CONF_FILE="${NGINX_FOLDER}/nginx/nginx.conf" NGINX_URL="${HS_FILES}/moonraker/nginx.tar.gz" NGINX_SERVICE_URL="${HS_FILES}/services/S50nginx" NGINX_CONF_URL="${HS_FILES}/moonraker/nginx.conf" - + # Supervisor Lite # SUPERVISOR_FILE="$BIN_FOLDER/supervisorctl" SUPERVISOR_OPT_FILE="/opt/bin/supervisorctl" @@ -210,7 +213,20 @@ function set_paths() { CREALITY_LOCAL_WEBRTC_SERVICE="${INITD_FOLDER}/CS59thirteenthp" CREALITY_AI_SERVICE="${INITD_FOLDER}/CS57solusp_service" CREALITY_MDNS_SERVICE="${INITD_FOLDER}/CS99mdns" - + + # Retire Nexusp Backend (K1C 2025) # + # INITD_FOLDER is already model-branched above; nexusp only exists on the + # K1_2025, and the option using these is only reachable from its Customize + # menu. Both prefixes are listed because the init script name varies by + # firmware, the same way tools.sh already probes for both forms. + NEXUSP_SERVICE="${INITD_FOLDER}/CS56nexusp_service" + NEXUSP_SERVICE_LEGACY="${INITD_FOLDER}/S56nexusp_service" + CREALITY_COMPAT_FILE="${MOONRAKER_FOLDER}/moonraker/moonraker/components/creality_compat.py" + CREALITY_COMPAT_URL="${HS_FILES}/moonraker/creality-compat/creality_compat.py" + MERGE_JOB_HISTORY_URL="${HS_FILES}/moonraker/creality-compat/merge_job_history.py" + MOONRAKER_DB="${PRINTER_DATA_FOLDER}/database/moonraker-sql.db" + NEXUSP_DB="${PRINTER_DATA_FOLDER}/database/nexusp-sql.db" + # Guppy Screen # GUPPY_SCREEN_FOLDER="${USR_DATA}/guppyscreen" GUPPY_SCREEN_URL1="${HS_FILES}/guppy-screen/guppy_update.cfg" diff --git a/scripts/retire_nexusp.sh b/scripts/retire_nexusp.sh new file mode 100644 index 0000000..67a76c6 --- /dev/null +++ b/scripts/retire_nexusp.sh @@ -0,0 +1,695 @@ +#!/bin/sh + +set -e + +# The K1C 2025 runs TWO Moonrakers against one Klipper: Creality's forked +# `nexusp` on :7125 (the touchscreen's backend) and the helper script's real one +# on :7126. They share -d /usr/data/printer_data, so one gcode directory and one +# klippy socket, but Creality namespaced the databases. +# +# That split is not merely redundant, it is a trap, and the failure mode is the +# bad kind: querying the wrong port does not fail, it ANSWERS. +# +# curl -s http://:7125/server/spoolman/status +# # nexusp -> {"error": {"code": 404, "message": "Method not found"}} +# +# Read at face value that says Spoolman was never connected on this printer. It +# is wrong, and every user who pastes a :7125 command from a Klipper forum gets +# a plausible-looking wrong answer instead of an error. +# +# This option retires nexusp and puts the real Moonraker on :7125 - the port the +# entire Klipper ecosystem assumes. The touchscreen is never patched: `vectorp` +# hardcodes http://127.0.0.1:7125 and we own what answers there, which is the +# whole trick, because the binary CANNOT be patched - it is a symlink into tmpfs, +# regenerated at boot from an encrypted SCBT blob on p8. +# +# What is load-bearing is not the daemon, it is two JSON-RPC methods the screen +# calls that stock Moonraker does not have. files/moonraker/creality-compat/ +# implements them; see the header of creality_compat.py. +# +# THE STEP ORDER IS NOT NEGOTIABLE, AND THE OBVIOUS ORDER IS WRONG +# +# 1 warn about anything that can restart Moonraker behind our back +# 2 stop Moonraker -- both must be down before step 4 +# 3 stop nexusp -- +# 4 merge the print history (dry run, then --apply; backs up first) +# 5 install the compat component + uncomment [creality_compat] +# 6 rename CS56nexusp_service -> disabled.CS56nexusp_service <- VERIFY +# 7 moonraker.conf: port 7126 -> 7125 +# 8 nginx.conf: server 127.0.0.1:7126; -> :7125; +# 9 start Moonraker, reload nginx explicitly +# 10 verify :7125 answers +# +# "Merge the history first, before anything is disabled" is the obvious order +# and it is impossible: merge_job_history.py refuses to run while any Moonraker +# or nexusp process is alive, because Moonraker caches job_totals in memory and +# writes its stale copy back at the next print. +# +# Step 6 must be VERIFIED before step 7 runs. If the rename fails and the port +# has already moved, nexusp and the real Moonraker both want :7125 and the loser +# dies silently. This uses the same guarded-mv-with-rollback idiom as +# creality_disable_one_service in disable_creality_services.sh, but deliberately +# does NOT call that function: CREALITY_SERVICES_NEVER_DISABLE lists both +# S56nexusp_service and CS56nexusp_service, so it would refuse - correctly, since +# that option does not own nexusp and this one does. +# +# TWO TRAPS WORTH STATING HERE RATHER THAN LEARNING TWICE +# +# - Any Moonraker SUPERVISOR must be stopped first, not just the daemon. The +# merge checks for a live daemon at one instant; a supervisor can restart +# Moonraker in the gap and the restarted daemon flushes its cached job_totals +# over the merged ones. This repo ships no watchdog, so upstream this is a +# warning rather than a step - a fork that adds one owns disarming it. +# - nginx reload. files/services/S50nginx runs "$NGINX" -s reload with no -c on +# its reload path, so nginx opens /etc/nginx/nginx.conf, which does not exist +# on this board, fails, and leaves the OLD config live while reporting +# nothing. That file is fixed in the same change as this one, but on boxes +# where Creality's own S50nginx won the `[ ! -f ]` guard at install time the +# shipped fix is not the file that runs - so reload explicitly here instead of +# trusting the init verb. + +NEXUSP_RETIRE_STAGE=0 + +function retire_nexusp_message(){ + top_line + title 'Retire Nexusp Backend' "${yellow}" + inner_line + hr + echo -e " │ ${cyan}The 2025 runs two Moonrakers against one Klipper: Creality's ${white}│" + echo -e " │ ${cyan}nexusp on port 7125 for the touchscreen, and the helper's on ${white}│" + echo -e " │ ${cyan}7126 for everything else. Querying the wrong one does not ${white}│" + echo -e " │ ${cyan}fail, it answers - wrongly. This retires nexusp, moves real ${white}│" + echo -e " │ ${cyan}Moonraker to 7125 and installs the two methods the screen ${white}│" + echo -e " │ ${cyan}needs. Your print history is merged, not replaced. ${white}│" + hr + bottom_line +} + +function restore_nexusp_message(){ + top_line + title 'Restore Nexusp Backend' "${yellow}" + inner_line + hr + echo -e " │ ${cyan}This puts Creality's nexusp back on port 7125 and returns the ${white}│" + echo -e " │ ${cyan}helper's Moonraker to 7126. Prints made while nexusp was ${white}│" + echo -e " │ ${cyan}retired are merged back into its database first, or the ${white}│" + echo -e " │ ${cyan}touchscreen's history would silently stop at the retire date. ${white}│" + hr + bottom_line +} + +# -------------------------------------------------------------------------- +# Predicates. Pure: they read the filesystem and say what they see, and nothing +# here mutates anything - so the menus and the info screen can call them freely. +# -------------------------------------------------------------------------- + +function nexusp_disabled_path() { + echo "$(dirname "$1")/disabled.$(basename "$1")" +} + +# Both the S and CS prefixes: the init script name varies by firmware, and +# tools.sh and tools_menu_K1C_2025.sh already probe for both forms elsewhere. +function nexusp_service_files() { + echo "$NEXUSP_SERVICE $NEXUSP_SERVICE_LEGACY" +} + +# The enabled init script, if there is one. Empty otherwise. +# +# The trailing `return 0` is load-bearing: without it a loop that finds nothing +# returns the last failed `[ -f ]`, and `svc="$(nexusp_enabled_service)"` would +# then abort the whole helper under the `set -e` helper.sh applies globally. +# Absence is an answer here, not an error. +function nexusp_enabled_service() { + local svc + for svc in $(nexusp_service_files); do + if [ -f "$svc" ]; then + echo "$svc" + return 0 + fi + done + return 0 +} + +# The disabled.* init script, if there is one. Empty otherwise. +function nexusp_disabled_service() { + local svc disabled_svc + for svc in $(nexusp_service_files); do + disabled_svc="$(nexusp_disabled_path "$svc")" + if [ -f "$disabled_svc" ]; then + echo "$disabled_svc" + return 0 + fi + done + return 0 +} + +function nexusp_present() { + [ -n "$(nexusp_enabled_service)" ] +} + +function nexusp_retired() { + [ -n "$(nexusp_disabled_service)" ] +} + +# True when this firmware ships no nexusp at all, in either form. Distinguishes +# "nothing to do here" from "already done", which look identical otherwise. +function nexusp_absent() { + if nexusp_present || nexusp_retired; then + return 1 + fi + return 0 +} + +# BOTH forms present. /usr/apps/etc/init.d survives a factory reset, but a +# firmware OTA can put CS56nexusp_service back beside the disabled copy. When +# that happens /etc/init.d/rcK starts it from the CS pass while +# S56moonraker_service starts from the S pass, so the real Moonraker wins the +# :7125 bind and nexusp dies silently at every boot - both files present, the +# box in a state nobody diagnoses. Reported by the menus, repaired by re-running +# the rename. +function nexusp_resurrected() { + if nexusp_present && nexusp_retired; then + return 0 + fi + return 1 +} + +# The port Moonraker is configured to listen on, or empty if unreadable. The +# pipe is deliberate: sed succeeds whether or not grep matched, so a config +# without a port line reads as "unknown" rather than aborting the helper. +function nexusp_moonraker_port() { + if [ ! -f "$MOONRAKER_CFG" ]; then + return 0 + fi + grep -m 1 '^port:' "$MOONRAKER_CFG" 2>/dev/null | sed 's/^port:[[:space:]]*//' +} + +# -------------------------------------------------------------------------- +# Small helpers shared by both directions +# -------------------------------------------------------------------------- + +# The Moonraker virtualenv's interpreter when it exists, so the merge runs on +# the same Python the daemon does; plain python3 otherwise (a workstation +# rehearsal, or a box where Moonraker was installed some other way). +function nexusp_python() { + if [ -x "$MOONRAKER_ENV_PYTHON" ]; then + echo "$MOONRAKER_ENV_PYTHON" + else + echo "python3" + fi +} + +function nexusp_pillow_installed() { + if [ ! -x "$MOONRAKER_ENV_PYTHON" ]; then + return 1 + fi + set +e + "$MOONRAKER_ENV_PYTHON" -c "import PIL" > /dev/null 2>&1 + local rc=$? + set -e + return $rc +} + +# The reload the init script's own verb cannot be trusted to do. See the header. +function nexusp_reload_nginx() { + local conf + echo -e "Info: Reloading Nginx..." + set +e + for conf in "$NGINX_CONF_FILE" /etc/nginx/nginx.conf; do + if [ -f "$conf" ] && [ -x "$NGINX_BIN" ]; then + "$NGINX_BIN" -c "$conf" -s reload > /dev/null 2>&1 && break + fi + done + set -e +} + +function nexusp_stop_service() { + local svc + svc="$(nexusp_enabled_service)" + if [ -z "$svc" ]; then + return + fi + echo -e "Info: Stopping nexusp..." + set +e + "$svc" stop > /dev/null 2>&1 + killall -q nexusp + set -e +} + +function nexusp_start_service() { + local svc + svc="$(nexusp_enabled_service)" + if [ -z "$svc" ]; then + return + fi + echo -e "Info: Starting nexusp..." + set +e + "$svc" start > /dev/null 2>&1 + set -e +} + +# Moonraker's port and Nginx's upstream, as one operation, because they must +# always agree - a box where they disagree serves 502s from Fluidd and nothing +# says why. Also used by configure_moonraker_nginx_k1_2025 so a Moonraker +# reinstall cannot silently undo the swap. +function nexusp_set_moonraker_port() { + local want="$1" other="$2" nginx_conf + if [ -f "$MOONRAKER_CFG" ]; then + echo -e "Info: Setting Moonraker port to ${want}..." + sed -i "s/^port:[[:space:]]*${other}\$/port: ${want}/" "$MOONRAKER_CFG" + fi + for nginx_conf in "$NGINX_CONF_FILE" /etc/nginx/nginx.conf; do + if [ -f "$nginx_conf" ]; then + echo -e "Info: Pointing Nginx Moonraker upstream to port ${want}..." + sed -i "s/server 127\.0\.0\.1:${other};/server 127.0.0.1:${want};/" "$nginx_conf" + fi + done +} + +# Link the component in and enable it. Follows moonraker_timelapse.sh exactly: +# the linked file is an untracked source file inside Moonraker's git repo, so +# update_manager reports "Repo has untracked source files" forever unless it is +# added to the repo's local exclude list. +function nexusp_install_compat_component() { + local repo_dir + echo -e "Info: Linking Creality compatibility component..." + ln -sf "$CREALITY_COMPAT_URL" "$CREALITY_COMPAT_FILE" + repo_dir="${CREALITY_COMPAT_FILE%/moonraker/components/creality_compat.py}" + if [ -d "$repo_dir"/.git/info ]; then + echo -e "Info: Excluding linked component from Moonraker repo..." + grep -qxF "moonraker/components/creality_compat.py" "$repo_dir"/.git/info/exclude 2>/dev/null \ + || echo "moonraker/components/creality_compat.py" >> "$repo_dir"/.git/info/exclude + fi + if [ ! -f "$MOONRAKER_CFG" ]; then + return + fi + if grep -q "^\[creality_compat\]" "$MOONRAKER_CFG"; then + echo -e "Info: [creality_compat] is already enabled in moonraker.conf file..." + elif grep -q "^#\[creality_compat\]" "$MOONRAKER_CFG"; then + echo -e "Info: Enabling [creality_compat] in moonraker.conf file..." + sed -i -e 's/^\s*#[[:space:]]*\[creality_compat\]/[creality_compat]/' -e '/^\[creality_compat\]/,/^\s*$/ s/^\(\s*\)#/\1/' "$MOONRAKER_CFG" + else + # A moonraker.conf written before this option existed has no block to + # uncomment. Without this branch the component would be linked and never + # loaded, so the screen's file browser would stay broken with nothing to + # show for it - and that is the majority case, since install_moonraker_nginx + # only rewrites moonraker.conf when Moonraker itself is reinstalled. + echo -e "Info: Adding [creality_compat] to moonraker.conf file..." + printf '\n[creality_compat]\ngenerate_thumbnails: True\nlog_requests: False\n' >> "$MOONRAKER_CFG" + fi +} + +function nexusp_remove_compat_component() { + local repo_dir + echo -e "Info: Removing Creality compatibility component..." + rm -f "$CREALITY_COMPAT_FILE" + rm -f "${CREALITY_COMPAT_FILE}c" + repo_dir="${CREALITY_COMPAT_FILE%/moonraker/components/creality_compat.py}" + if [ -f "$repo_dir"/.git/info/exclude ]; then + sed -i '/^moonraker\/components\/creality_compat\.py$/d' "$repo_dir"/.git/info/exclude + fi + if [ -f "$MOONRAKER_CFG" ] && grep -q "^\[creality_compat\]" "$MOONRAKER_CFG"; then + echo -e "Info: Disabling [creality_compat] in moonraker.conf file..." + sed -i '/^\[creality_compat\]/,/^\s*$/ s/^\(\s*\)\([^#]\)/#\1\2/' "$MOONRAKER_CFG" + fi +} + +# Everything install_moonraker_nginx would undo. It rm -f's moonraker.conf and +# re-copies the shipped one - which has [creality_compat] commented and port +# 7125 - then calls configure_moonraker_nginx_k1_2025, which used to sed the +# port back to 7126 unconditionally. On a retired box that left NOTHING +# answering :7125 and the touchscreen dead with no error anywhere. +function nexusp_reapply_retired_config() { + nexusp_set_moonraker_port 7125 7126 + nexusp_install_compat_component +} + +# The merge, in either direction, with the dry run shown first. Returns non-zero +# when it refuses; helper.sh sets -e globally and sources every script into the +# same shell, so an unguarded call would abort the whole helper and drop the +# user to a shell with no menu. Guarded at every call site. +function nexusp_merge_history() { + local direction="$1" python + python="$(nexusp_python)" + if [ ! -f "$MOONRAKER_DB" ] || [ ! -f "$NEXUSP_DB" ]; then + echo -e "Info: Only one print history database exists, nothing to merge..." + return 0 + fi + echo -e "Info: Print history merge (${direction}), dry run..." + set +e + "$python" "$MERGE_JOB_HISTORY_URL" --direction "$direction" + local rc=$? + if [ "$rc" != "0" ]; then + set -e + return "$rc" + fi + echo -e "Info: Merging print history..." + "$python" "$MERGE_JOB_HISTORY_URL" --direction "$direction" --apply + rc=$? + set -e + return "$rc" +} + +# -------------------------------------------------------------------------- +# Rollback. Undoes stages 6-8 in reverse, and is only ever called from the +# failure paths below - a partial swap is the one outcome worth more code than +# the swap itself, because the symptom is a dead touchscreen and nothing in any +# log to connect it to this option. +# -------------------------------------------------------------------------- + +function nexusp_rollback_retire() { + local disabled_svc svc + echo + echo -e "${yellow}Rolling back...${white}" + if [ "$NEXUSP_RETIRE_STAGE" -ge 8 ]; then + nexusp_set_moonraker_port 7126 7125 + fi + if [ "$NEXUSP_RETIRE_STAGE" -ge 6 ]; then + disabled_svc="$(nexusp_disabled_service)" + if [ -n "$disabled_svc" ]; then + svc="$(dirname "$disabled_svc")/$(basename "$disabled_svc" | sed 's/^disabled\.//')" + mv "$disabled_svc" "$svc" 2>/dev/null || true + fi + fi + if [ "$NEXUSP_RETIRE_STAGE" -ge 5 ]; then + nexusp_remove_compat_component + fi + nexusp_start_service + start_moonraker + nexusp_reload_nginx + # The forward history merge is NOT undone, deliberately. Those rows are valid + # Moonraker rows either way, and rolling them back would delete records that + # exist in no other database - the exact loss this whole option is careful + # about. Nothing else read them, so leaving them costs nothing. + error_msg "Nexusp has NOT been retired - the printer is back as it was." +} + +# -------------------------------------------------------------------------- +# Repair after a firmware update put the service file back +# -------------------------------------------------------------------------- + +# Re-applies the rename, keeping the file the firmware just wrote. Offered from +# retire_nexusp because that is the option a user reaches for when the menus +# report the state, and it is the same rename either way. `mv -f` rather than a +# delete: the two files are the same firmware init script, and one move leaves +# exactly one copy instead of briefly leaving none. +function nexusp_repair_resurrection() { + local svc disabled_svc + svc="$(nexusp_enabled_service)" + disabled_svc="$(nexusp_disabled_path "$svc")" + if ! creality_confirm_printer_idle; then + return + fi + nexusp_stop_service + echo -e "Info: Re-applying the nexusp rename..." + if ! mv -f "$svc" "$disabled_svc" 2>/dev/null; then + error_msg "Could not rename $(basename "$svc") - is $(dirname "$svc") writable?" + return + fi + if [ -f "$svc" ] || [ ! -f "$disabled_svc" ]; then + error_msg "The nexusp service did not stay renamed!" + return + fi + # The same update may also have reverted the port or the component. + nexusp_reapply_retired_config + echo -e "Info: Restarting Moonraker service..." + stop_moonraker + start_moonraker + nexusp_reload_nginx + ok_msg "The nexusp service has been disabled again!" + echo -e " ${cyan}Nothing else was changed; your history and settings are as they${white}" + echo -e " ${cyan}were before the firmware update.${white}" +} + +# -------------------------------------------------------------------------- +# Retire +# -------------------------------------------------------------------------- + +function retire_nexusp(){ + retire_nexusp_message + echo + echo -e " ${yellow}Warning: this changes which process answers the port the" + echo -e " touchscreen depends on. Do it with the printer idle. The nexusp" + echo -e " binary and its database are never deleted, so Restore Nexusp" + echo -e " Backend puts everything back.${white}" + echo + local yn pillow_yn repair_yn svc disabled_svc answered + NEXUSP_RETIRE_STAGE=0 + while true; do + read -p "${white} Are you sure you want to retire ${green}Nexusp Backend ${white}? (${yellow}y${white}/${yellow}n${white}): ${yellow}" yn + case "${yn}" in + Y|y) + echo -e "${white}" + if [ ! -d "$MOONRAKER_FOLDER" ]; then + error_msg "Moonraker is needed, please install it first!" + return + fi + # A firmware update can put the service file back beside the disabled + # copy. /etc/init.d/rcK then starts it from the CS pass while + # S56moonraker_service starts from the S pass, so Moonraker wins the + # :7125 bind and nexusp dies silently at every boot - both files + # present, and nothing anywhere says why the touchscreen "randomly + # stopped working after an update". + if nexusp_resurrected; then + echo -e " ${yellow}Both the nexusp service and its disabled copy exist. A firmware" + echo -e " update recreated it, and it now loses the race for port 7125 to" + echo -e " Moonraker at every boot. The repair is to re-apply the rename," + echo -e " keeping the file the update wrote.${white}" + echo + read -p " ${white}Disable the recreated ${green}nexusp service ${white}again? (${yellow}y${white}/${yellow}n${white}): ${yellow}" repair_yn + echo -e "${white}" + case "${repair_yn}" in + Y|y) + nexusp_repair_resurrection;; + *) + error_msg "Repair canceled!";; + esac + return + fi + if ! nexusp_present; then + if nexusp_retired; then + error_msg "Nexusp Backend is already retired!" + else + error_msg "No nexusp service was found on this firmware!" + fi + return + fi + if ! creality_confirm_printer_idle; then + return + fi + + # Every decision is collected BEFORE the first mutation, so an abandoned + # prompt cannot leave the printer half-swapped. + pillow_yn="n" + if ! nexusp_pillow_installed; then + echo -e " ${yellow}Pillow is not in Moonraker's virtualenv. Without it the screen" + echo -e " shows no thumbnail at all for newly uploaded files - and that is" + echo -e " true of Moonraker's own thumbnail parsing today, retired or not," + echo -e " so installing it fixes both. It is a large download and there may" + echo -e " be no prebuilt wheel for this board.${white}" + echo + read -p " ${white}Install ${green}Pillow ${white}into Moonraker's virtualenv? (${yellow}y${white}/${yellow}n${white}): ${yellow}" pillow_yn + echo -e "${white}" + fi + + # Re-check immediately before mutating: the prompt above may have taken + # a while, and a print may have been started from the screen. + if ! creality_confirm_printer_idle; then + return + fi + + case "${pillow_yn}" in + Y|y) + echo -e "Info: Installing Pillow..." + set +e + "$MOONRAKER_ENV_PYTHON" -m pip install Pillow + if [ "$?" != "0" ]; then + echo -e "${yellow}Warning: Pillow could not be installed. Continuing without it -${white}" + echo -e "${yellow}thumbnails already on disk are still listed.${white}" + fi + set -e;; + esac + + NEXUSP_RETIRE_STAGE=1 + # 1. Anything that can restart Moonraker behind our back has to be off + # BEFORE the merge, not just the daemon. Nothing in this repo can, so + # this is a warning; a fork that ships a watchdog owns disarming it. + echo -e "Info: If you run a Moonraker watchdog or supervisor, stop it now -" + echo -e " a restart mid-merge overwrites the merged totals." + # 2-3. Both daemons down, or the merge refuses. + echo -e "Info: Stopping Moonraker service..." + stop_moonraker + NEXUSP_RETIRE_STAGE=2 + nexusp_stop_service + NEXUSP_RETIRE_STAGE=3 + + # 4. The merge. Nothing has been renamed or re-pointed yet, so a refusal + # here just puts the daemons back. + set +e + nexusp_merge_history to-moonraker + if [ "$?" != "0" ]; then + set -e + error_msg "The print history merge refused to run - nothing was changed." + echo -e " ${darkred}See the message above. Retiring nexusp without it would leave${white}" + echo -e " ${darkred}the screen with a history that starts on the day you installed${white}" + echo -e " ${darkred}the helper script.${white}" + nexusp_start_service + start_moonraker + return + fi + set -e + NEXUSP_RETIRE_STAGE=4 + + # 5. The component. Safe to install while nexusp is still enabled: it is + # inert until Moonraker loads it. + nexusp_install_compat_component + NEXUSP_RETIRE_STAGE=5 + + # 6. The rename, guarded. An unguarded mv would abort the whole helper + # under set -e, leaving both daemons stopped with no message and no + # menu to return to. + svc="$(nexusp_enabled_service)" + disabled_svc="$(nexusp_disabled_path "$svc")" + if [ -f "$disabled_svc" ]; then + error_msg "Both $(basename "$svc") and $(basename "$disabled_svc") exist!" + echo -e " ${darkred}A firmware update likely recreated it. Leaving both alone to${white}" + echo -e " ${darkred}avoid losing the backup. Delete whichever copy you do not want${white}" + echo -e " ${darkred}and run this option again.${white}" + nexusp_rollback_retire + return + fi + echo -e "Info: Disabling nexusp service..." + if ! mv "$svc" "$disabled_svc" 2>/dev/null; then + error_msg "Could not rename $(basename "$svc") - is $(dirname "$svc") writable?" + nexusp_rollback_retire + return + fi + # VERIFIED, not assumed. If the rename silently did not take and the + # port moves anyway, both daemons want :7125 and the loser dies with + # nothing in any log. + if [ -f "$svc" ] || [ ! -f "$disabled_svc" ]; then + error_msg "The nexusp service did not stay renamed!" + nexusp_rollback_retire + return + fi + NEXUSP_RETIRE_STAGE=6 + + # 7-8. The port, on both sides at once. + nexusp_set_moonraker_port 7125 7126 + NEXUSP_RETIRE_STAGE=8 + + # 9. Start, and reload nginx explicitly rather than trusting the verb. + echo -e "Info: Starting Moonraker service..." + start_moonraker + nexusp_reload_nginx + NEXUSP_RETIRE_STAGE=9 + + # 10. Verify something actually answers on the port the screen polls. + set +e + "$CURL" -s -m 5 http://127.0.0.1:7125/server/info | grep -q '"result"' + answered=$? + set -e + if [ "$answered" != "0" ]; then + error_msg "Nothing answered on port 7125 after the swap!" + nexusp_rollback_retire + return + fi + + ok_msg "Nexusp Backend has been retired successfully!" + echo -e " ${cyan}Moonraker now answers on 7125 for the touchscreen and for you.${white}" + echo -e " ${cyan}The touchscreen reconnects on its own; give it a few seconds.${white}" + echo -e " ${cyan}For about four seconds after a COLD boot the screen polls two${white}" + echo -e " ${cyan}methods Moonraker only registers once Klipper connects. It${white}" + echo -e " ${cyan}resolves itself and needs no action.${white}" + echo -e " ${cyan}A firmware update can put the nexusp service back - the menus${white}" + echo -e " ${cyan}report it, and re-running this option repairs it.${white}" + return;; + N|n) + error_msg "Retiring canceled!" + return;; + *) + error_msg "Please select a correct choice!";; + esac + done +} + +# -------------------------------------------------------------------------- +# Restore +# -------------------------------------------------------------------------- + +function restore_nexusp(){ + restore_nexusp_message + echo + echo -e " ${yellow}Warning: while nexusp was retired the touchscreen wrote its" + echo -e " history to Moonraker's database. Those prints are merged back into" + echo -e " nexusp's before it starts, so this takes longer than it sounds -" + echo -e " skipping it would make every print since the swap disappear from" + echo -e " the screen with no warning.${white}" + echo + local yn svc disabled_svc + while true; do + restore_msg "Nexusp Backend" yn + case "${yn}" in + Y|y) + echo -e "${white}" + if ! nexusp_retired; then + error_msg "Nexusp Backend is not retired!" + return + fi + if ! creality_confirm_printer_idle; then + return + fi + + echo -e "Info: If you run a Moonraker watchdog or supervisor, stop it now -" + echo -e " a restart mid-merge overwrites the merged totals." + echo -e "Info: Stopping Moonraker service..." + stop_moonraker + + # The merge, BACKWARDS, while both are stopped. This must NOT try to + # un-merge the forward direction: those rows are valid Moonraker rows + # either way and rolling them back would delete records that have no + # other copy. + set +e + nexusp_merge_history to-nexusp + if [ "$?" != "0" ]; then + set -e + error_msg "The print history merge refused to run - nothing was changed." + echo -e " ${darkred}See the message above. Restoring without it would hand the${white}" + echo -e " ${darkred}screen a history frozen at the day you retired nexusp.${white}" + start_moonraker + return + fi + set -e + + # Reverse of 8, then 7, then 6. + nexusp_set_moonraker_port 7126 7125 + nexusp_remove_compat_component + disabled_svc="$(nexusp_disabled_service)" + svc="$(dirname "$disabled_svc")/$(basename "$disabled_svc" | sed 's/^disabled\.//')" + if [ -f "$svc" ]; then + echo -e "${yellow}Warning: $(basename "$svc") already exists - the firmware recreated it.${white}" + echo -e "${yellow}Keeping the newer file; $(basename "$disabled_svc") left in place.${white}" + else + echo -e "Info: Restoring nexusp service..." + if ! mv "$disabled_svc" "$svc" 2>/dev/null; then + error_msg "Could not restore $(basename "$svc") - is $(dirname "$svc") writable?" + start_moonraker + return + fi + fi + + echo -e "Info: Starting Moonraker service..." + start_moonraker + nexusp_start_service + nexusp_reload_nginx + ok_msg "Nexusp Backend has been restored successfully!" + echo -e " ${cyan}The touchscreen is back on nexusp, and Moonraker on 7126.${white}" + return;; + N|n) + error_msg "Restoration canceled!" + return;; + *) + error_msg "Please select a correct choice!";; + esac + done +} From 47c6d8c80f62aa86960506b18446093f759d5225 Mon Sep 17 00:00:00 2001 From: arlophoenix Date: Mon, 3 Aug 2026 18:30:15 +1200 Subject: [PATCH 3/4] Harden the nexusp retirement after pre-landing review Fixes found by a seven-specialist review of the previous commit, three of them verified by execution rather than inspection. The helper died mid-retirement. nexusp_merge_history re-enabled errexit before returning non-zero, and because helper.sh sources every script into one shell under a global set -e, that killed the whole helper at the call site - past the guard meant to catch it, with both daemons stopped and no menu to return to. Verified failing in bash, sh, dash and zsh. The history merge lost prints and mis-ordered the rest. Duplicate matching collapsed several source rows onto one target row, so a cancelled print and its retry became two duplicates and one real record vanished; matching is now one-to-one, nearest first. And Moonraker pages history with ORDER BY job_id, not start_time, so appended older prints came back presented as the newest - rows are now renumbered into start_time order. The screen could be left with nothing answering it. Step 10 checked only that port 7125 replied, but Moonraker swallows optional-component load failures and keeps serving, so a broken shim reported success with a dead file browser. restore_nexusp moved the port before the rename that could fail, and remove_moonraker_nginx had no retired-box guard at all. get_directory_ex dropped disk_usage, which is present in all ten golden nexusp captures and is where the screen reads its free-space figure. The .thumbs reserved-path trick is gone. It suppressed phantom notifications by making thumbnails permanently undeletable through Fluidd and taxing every Moonraker listing; renders are capped per request instead. Adds a shell test suite for the option (20 cases) and takes the Python suites to 119. Merge now confirms before applying, validates the schema it writes into, and refuses on an instance_id mismatch. --- README.md | 34 +- .../creality-compat/creality_compat.py | 228 +++++---- .../creality-compat/merge_job_history.py | 255 +++++++++- .../creality-compat/test_creality_compat.py | 115 ++--- .../creality-compat/test_merge_job_history.py | 274 ++++++++++- scripts/menu/K1_2025/info_menu_K1C_2025.sh | 8 +- scripts/moonraker_nginx.sh | 38 +- scripts/retire_nexusp.sh | 346 +++++++++++--- tests/test_retire_nexusp.py | 446 ++++++++++++++++++ 9 files changed, 1473 insertions(+), 271 deletions(-) create mode 100644 tests/test_retire_nexusp.py diff --git a/README.md b/README.md index d390671..89e40d5 100644 --- a/README.md +++ b/README.md @@ -58,17 +58,35 @@ history would silently stop at the day you retired it. - Two of the four Creality-only RPC methods are deliberately not implemented: `server.history.debug.job` and `server.debug.status`. Neither is screen-facing. +- **The two shimmed methods answer over the websocket only.** `curl + http://:7125/server/files/get_directory_ex` returns 404 by design — + that is what nexusp did, and matching it is the point. Note that this 404 is + identical to the one you get when the component is not loaded at all, so it + is not a way to check. To confirm the shim is live: -### Running the component's tests + ```sh + curl -s http://:7125/server/info + ``` -The component and the history merge ship with their tests beside them. They are -the executable record of what was measured against `nexusp` before it was -switched off — once it is retired those measurements cannot be re-derived -without reviving it. They need only Python and pytest, no printer and no -Moonraker: + and look for `creality_compat` in `components` (and *not* in + `failed_components`). + +### Running the tests + +Everything ships with tests beside it. They are the executable record of what +was measured against `nexusp` before it was switched off — once it is retired +those measurements cannot be re-derived without reviving it. No printer, no +Moonraker, no network: ```sh -python3 -m pytest -q files/moonraker/creality-compat +python3 -m pytest -q ``` -This repository has no CI, so nothing runs them automatically. +That runs both suites: the Moonraker component and history merge under +`files/moonraker/creality-compat/`, and the shell option under `tests/`. The +component suite also runs standalone from its own directory with no conftest. + +The shell tests exercise `sed -i` the way the printer does, which is GNU/busybox +syntax; on macOS they skip unless `gsed` is installed (`brew install gnu-sed`). + +This repository has no CI, so nothing runs any of this automatically. diff --git a/files/moonraker/creality-compat/creality_compat.py b/files/moonraker/creality-compat/creality_compat.py index 4c75643..09615eb 100644 --- a/files/moonraker/creality-compat/creality_compat.py +++ b/files/moonraker/creality-compat/creality_compat.py @@ -57,7 +57,15 @@ # on the reference in this file. # # `keyword` is a case-insensitive substring match on the name, and it narrows -# `count` as well as the page. +# `count` as well as the page. It applies to FILES ONLY — subdirectories are +# listed whatever the keyword, per the rule above. +# +# That last part is the one behaviour in this file chosen rather than measured. +# The ten golden captures happen not to cover it: the only keyword case has no +# subdirectories in it and the only directory case sends no keyword, so what +# nexusp did here is genuinely unknown. Of the two readings, this is the one +# that cannot make something vanish from the browser — a folder disappearing +# while you type is indistinguishable from a folder that was deleted. # # `since` and `before` are accepted and DELIBERATELY IGNORED, because that is # what nexusp does. Measured, not assumed: a window excluding almost every file @@ -227,10 +235,26 @@ THUMB_DIR = ".thumbs" THUMB_RE = re.compile(r"^(?P.+)-(?P\d+)x(?P\d+)\.png$") -# Bound on the load-time walk that finds `.thumbs` directories to reserve. A -# gcodes root deep or wide enough to exceed this is not a printer this option -# was measured on, and the walk runs before Moonraker serves anything. -MAX_RESERVE_WALK_DIRS = 2000 +# Renders per request, across the whole page. +# +# WRITING A PNG INTO `.thumbs` MAKES MOONRAKER BROADCAST A PHANTOM create_file. +# file_manager watches every directory under the gcodes root and has no dot +# directory filter, so each generated thumbnail is announced to Fluidd and to +# the screen as a new file appearing. An earlier version suppressed that by +# calling `file_manager.add_reserved_path` on each `.thumbs` directory - which +# works, and costs far too much: `check_reserved_path(path, need_write=True)` +# guards delete/move/upload, so a reserved `.thumbs` becomes permanently +# UNDELETABLE through Fluidd on a printer where clearing thumbnails is exactly +# what people do for space, there is no removal API for a reservation, and +# `get_path_info` linear scans the reserved list once per directory entry so +# every listing in Moonraker pays for it forever. +# +# So the notifications are accepted instead, and merely bounded: generation +# already happens once per file for the life of that file, and this cap stops a +# single large page turning into a burst. The events are transient and +# self-limiting; the reservation's costs were permanent and fell on people not +# using this option. +MAX_RENDERS_PER_REQUEST = 8 # The screen sends commas (`name,asc,folder`). The rest are accepted because the # cost of another separator turning up is a sort that silently does nothing. @@ -279,6 +303,18 @@ def __init__(self, config: ConfigHelper) -> None: "file browser needs it. Report it against the helper " "script rather than editing this file blind.", 500 ) + # Coupling 3 checked here too, not just its table name. Importing + # HIST_TABLE catches a renamed CONSTANT; it does not catch a renamed or + # restructured `history_table` ATTRIBUTE, which would load clean, appear + # healthy in /server/info, pass the option's own verification, and then + # raise the first time the screen asks for a count. + history = self.server.lookup_component("history") + if not hasattr(history, "history_table"): + raise self.server.error( + "creality_compat: history component has no 'history_table'. " + "This Moonraker is newer than the component; server.history." + "count needs it.", 500 + ) # Pillow, once, here — not per request. Missing PIL must cost one log # line at startup, not a traceback per file forever. @@ -287,9 +323,6 @@ def __init__(self, config: ConfigHelper) -> None: # because it encodes directory, stem and size. In memory only, so a # repaired filesystem heals at the next Moonraker restart. self._failed_renders: Set[str] = set() - self._reserved_thumb_dirs: Set[str] = set() - if self.generate_thumbs: - self._reserve_existing_thumb_dirs(fm) # WEBSOCKET only, deliberately. `register_endpoint` defaults to every # transport, which would answer `GET /server/files/directory_ex` over @@ -334,46 +367,14 @@ def _load_pillow(self) -> Optional[Any]: return None return Image - def _reserve_thumb_dir(self, fm: Any, thumb_dir: str) -> None: - """Keep `.thumbs` out of file_manager's inotify watch. - - Writing a PNG into a watched directory fires `notify_filelist_changed`, - and file_manager has no dot-directory filter — so generating thumbnails - inside a listing would broadcast a phantom `create_file` for every PNG - to Fluidd and to the screen. A reserved path is skipped both by the - initial scan and by the directory-create handler, and read access is - left on so the thumbnails are still served over HTTP. - - This must happen BEFORE the directory is scanned or created, which is - why the existing ones are reserved at load (file_manager's initial scan - runs in its `component_init`, after every component is constructed) and - a new one is reserved before it is made. Both `add_reserved_path` and - `get_directory` are public API; a failure here is cosmetic, so it is - logged and swallowed rather than raised. - """ - if thumb_dir in self._reserved_thumb_dirs: - return - self._reserved_thumb_dirs.add(thumb_dir) - try: - fm.add_reserved_path(f"creality_compat:{thumb_dir}", thumb_dir, True) - except Exception: - logging.exception( - "creality_compat: could not reserve %s; thumbnail writes there " - "will emit spurious filelist notifications", thumb_dir - ) - def _ensure_thumb_dir(self, dir_path: str) -> bool: - """Reserve `/.thumbs`, then make sure it exists. Order matters. + """Make sure `/.thumbs` exists, and say whether it does. - Reserving first is what stops the directory-create event from starting a - watch on it, which is what stops every PNG written afterwards from - broadcasting a phantom `create_file`. Called only when there is actually - something to render, so a directory of thumbnail-less files never grows - an empty `.thumbs`. + Called only when there is actually something to render, so a directory + of thumbnail-less files never grows an empty `.thumbs` just because it + was browsed. """ thumb_dir = os.path.join(dir_path, THUMB_DIR) - self._reserve_thumb_dir( - self.server.lookup_component("file_manager"), thumb_dir) if os.path.isdir(thumb_dir): return True try: @@ -386,28 +387,6 @@ def _ensure_thumb_dir(self, dir_path: str) -> bool: return False return True - def _reserve_existing_thumb_dirs(self, fm: Any) -> None: - try: - gcode_root = fm.get_directory("gcodes") - except Exception: - gcode_root = "" - if not gcode_root or not os.path.isdir(gcode_root): - return - seen = 0 - for dir_path, subdirs, _ in os.walk(gcode_root): - seen += 1 - if seen > MAX_RESERVE_WALK_DIRS: - logging.info( - "creality_compat: stopped reserving .thumbs directories " - "after %d directories; deeper ones will emit spurious " - "filelist notifications when a thumbnail is written", - MAX_RESERVE_WALK_DIRS - ) - return - if THUMB_DIR in subdirs: - subdirs.remove(THUMB_DIR) - self._reserve_thumb_dir(fm, os.path.join(dir_path, THUMB_DIR)) - def _history_instance(self, history: Any) -> str: """The instance id `server.history.list` scopes to. @@ -482,7 +461,8 @@ def _disk_thumbnails(self, dir_path: str) -> Dict[str, List[Dict[str, Any]]]: return found def _generate_missing( - self, dir_path: str, stem: str, have: List[Dict[str, Any]] + self, dir_path: str, stem: str, have: List[Dict[str, Any]], + budget: List[int] ) -> List[Dict[str, Any]]: """Render the sizes nexusp used to render, from the biggest one present. @@ -502,22 +482,28 @@ def _generate_missing( """ if not self.generate_thumbs or self._image is None: return [] - source = None - for thumb in have: - if source is None or thumb["width"] * thumb["height"] > \ - source["width"] * source["height"]: - source = thumb - if source is None: + # Largest first, and a LIST rather than one pick. Moonraker's metadata + # lists thumbnails it parsed out of the gcode, whose PNGs may no longer + # be on disk — clearing `.thumbs` for space is exactly what people do on + # this printer. With a single pick, one stale metadata entry that + # happens to be the biggest sends both destinations to _failed_renders + # for the life of the process while a perfectly good smaller source sits + # beside it. + sources = sorted(have, key=lambda t: t["width"] * t["height"], reverse=True) + if not sources: return [] + source = sources[0] wanted = [ (width, height) for width, height in GENERATED_THUMB_SIZES if not any(t["width"] == width and t["height"] == height for t in have) and source["width"] >= width and source["height"] >= height ] - if not wanted or not self._ensure_thumb_dir(dir_path): + if not wanted or budget[0] <= 0 or not self._ensure_thumb_dir(dir_path): return [] made: List[Dict[str, Any]] = [] for width, height in wanted: + if budget[0] <= 0: + break name = f"{stem}-{width}x{height}.png" dest = os.path.join(dir_path, THUMB_DIR, name) # One attempt per destination per process. Retrying a render that @@ -526,30 +512,64 @@ def _generate_missing( # on its own. if dest in self._failed_renders: continue - try: - with self._image.open( - os.path.join(dir_path, source["relative_path"]) - ) as im: - im.convert("RGBA").resize( - (width, height), self._image.LANCZOS).save(dest) + budget[0] -= 1 + if self._render(dir_path, sources, width, height, dest): made.append({ "width": width, "height": height, "size": os.path.getsize(dest), "relative_path": f"{THUMB_DIR}/{name}", }) - except Exception as why: - # A directory listing must not fail because one PNG would not - # scale. One warning per destination, then silence. - self._failed_renders.add(dest) - logging.warning( - "creality_compat: could not render %s (%s); not trying " - "again until Moonraker restarts", dest, why - ) return made + def _render(self, dir_path: str, sources: List[Dict[str, Any]], + width: int, height: int, dest: str) -> bool: + """Downscale the first source that actually opens, ATOMICALLY. + + Written to a temp name in the same directory and `os.replace`d onto the + destination. A direct write is visible half-finished: two concurrent + get_directory_ex calls decorate in separate executor threads and can + target the same file, and a power cut mid-write does the same. The + result would be a truncated PNG that is never repaired — `_failed_renders` + only remembers exceptions, and on the next listing the file exists with + the right `-WxH.png` suffix, so the size counts as satisfied forever. + The temp name has no `-WxH.png` suffix, so THUMB_RE cannot match it even + if a crash leaves one behind. + """ + last_error = None + for source in sources: + # Never upscale, whichever source we fall back to. A 32x32 blown up + # into a 195x195 tile reads as a BROKEN thumbnail, which is worse + # than an absent one — and the eligibility check above only vetted + # the largest source, so a fallback has to be re-checked here. + if source["width"] < width or source["height"] < height: + continue + src_path = os.path.join(dir_path, source["relative_path"]) + tmp = f"{dest}.part" + try: + with self._image.open(src_path) as im: + im.convert("RGBA").resize( + (width, height), self._image.LANCZOS).save(tmp) + os.replace(tmp, dest) + return True + except Exception as why: + last_error = why + try: + os.remove(tmp) + except OSError: + pass + # A directory listing must not fail because no PNG would scale. One + # warning per destination, then silence until Moonraker restarts. + self._failed_renders.add(dest) + logging.warning( + "creality_compat: could not render %s from any of %d source(s) " + "(%s); not trying again until Moonraker restarts", + dest, len(sources), last_error + ) + return False + def _merge_thumbnails( self, entry: Dict[str, Any], on_disk: Dict[str, List[Dict[str, Any]]], - dir_path: str + dir_path: str, budget: List[int] ) -> None: """Add the on-disk thumbnails this file has that the metadata omits. @@ -564,7 +584,7 @@ def _merge_thumbnails( have = {(t.get("width"), t.get("height")) for t in thumbs} thumbs.extend(t for t in on_disk.get(stem, []) if (t["width"], t["height"]) not in have) - thumbs.extend(self._generate_missing(dir_path, stem, thumbs)) + thumbs.extend(self._generate_missing(dir_path, stem, thumbs, budget)) if not thumbs: return entry["thumbnails"] = sorted(thumbs, key=lambda t: t["width"] * t["height"]) @@ -582,8 +602,14 @@ def _decorate_page(self, dir_path: str, page: List[Dict[str, Any]]) -> None: if not files: return on_disk = self._disk_thumbnails(dir_path) + # A one-element list rather than an int so the count is shared by + # reference across every file on the page. Each generated PNG makes + # Moonraker broadcast a create_file, so the cap bounds the burst a + # single large page can produce; the files past it simply get their + # thumbnails on a later listing. + budget = [MAX_RENDERS_PER_REQUEST] for entry in files: - self._merge_thumbnails(entry, on_disk, dir_path) + self._merge_thumbnails(entry, on_disk, dir_path, budget) def _sorted(self, items: List[Dict[str, Any]], order: str) -> List[Dict[str, Any]]: # Case-folded, and split on commas/semicolons as well as whitespace: the @@ -628,8 +654,12 @@ async def _handle_directory_ex(self, web_request: WebRequest) -> Dict[str, Any]: dirs: List[Dict[str, Any]] = [] files: List[Dict[str, Any]] = [] for entry in listing["dirs"]: + # Directories are NOT keyword-filtered. See the header: the rule is + # that subdirectories are always listed, and a search that hides the + # folder you were about to open is the same "it looks deleted" + # failure this shim exists to avoid. name = entry.get("dirname", "") - if self._visible(name, True) and keyword in name.lower(): + if self._visible(name, True): dirs.append(dict(entry, type="d")) for entry in listing["files"]: name = entry.get("filename", "") @@ -650,7 +680,19 @@ async def _handle_directory_ex(self, web_request: WebRequest) -> Dict[str, Any]: # only reader and it has always been given one. root_info.setdefault("name", root) root_info["path"] = path - return {"items": page, "count": count, "root_info": root_info} + # `disk_usage` is in every one of the ten golden nexusp captures + # (`{"count", "disk_usage", "items", "root_info"}`), and an earlier + # version of this shim dropped it. Moonraker computes it in + # `_list_directory` for free, and a client reading a key that is now + # absent shows a blank or stale free-space figure with no error - the + # silent-wrong-answer failure this whole option exists to remove. Extra + # keys are ignored by JSON clients; missing ones are not. + return { + "items": page, + "count": count, + "disk_usage": listing.get("disk_usage", {}), + "root_info": root_info, + } # -- server.history.count ----------------------------------------------- diff --git a/files/moonraker/creality-compat/merge_job_history.py b/files/moonraker/creality-compat/merge_job_history.py index 0fbb6d7..45c6815 100644 --- a/files/moonraker/creality-compat/merge_job_history.py +++ b/files/moonraker/creality-compat/merge_job_history.py @@ -101,11 +101,25 @@ # warning rather than a step; a fork that adds one owns disarming it. # - Backs the target up next to itself before the first write, and prints the # command that restores it. That file is the entire rollback story. -# - Does NOT renumber job_id. Inserted rows take ids after the target's existing -# ones, so id order no longer matches time order — deliberately. Nothing joins -# on job_id and every surface sorts by start_time, whereas renumbering rewrites -# the identity of rows a client may already be holding: a Fluidd tab left open -# across the merge would delete by an id that now names a different print. +# - RENUMBERS job_id into start_time order, in the same transaction as the +# inserts. An earlier version deliberately did not, on the stated grounds that +# "nothing joins on job_id and every surface sorts by start_time". The second +# half of that is false, and checkably so — Moonraker's own history list is: +# +# sql_statement += f" ORDER BY job_id {order}" # history.py, order="desc" +# +# with no ORDER BY start_time anywhere in the file. Appended rows take the +# highest ids, so without renumbering the recovered prints — the OLDEST on the +# machine, which is the entire point of this merge — come back as the most +# recent jobs in Fluidd and on the touchscreen, while server.history.count +# reports the right total. A correct scrollbar over a wrongly ordered list is +# exactly the plausible-wrong-answer failure retiring nexusp exists to remove. +# +# What renumbering costs is the identity of rows a client may already hold: a +# Fluidd tab left open across the merge could delete by an id that now names a +# different print. That needs a tab open across a daemon restart AND a delete +# issued into the seconds before the port moves and Moonraker comes back. The +# mis-ordering it prevents is permanent and visible to everyone. import argparse import os @@ -142,21 +156,29 @@ "total_filament_used", "longest_job", "longest_print") -def moonraker_running(): - """True if anything that looks like Moonraker holds a PID right now. +def moonraker_running(proc="/proc"): + """The cmdline of a live Moonraker or nexusp process, or None. + + None means BOTH "nothing is running" and "there is no /proc to look in" — + the latter being a dry-run rehearsal against copied databases on a + workstation, where there is no daemon to collide with. The caller cannot + tell those apart and does not need to. /proc scan rather than pgrep: busybox ps on this board truncates the command line at a width that hides moonraker.py behind the venv python path. + + `proc` is an argument only so the scan itself can be tested against a fake + tree — this is the single guard standing between a live daemon and an + irreversible write, and stubbing the whole function out (which every other + test does) leaves the matching logic never executed. """ - if not os.path.isdir("/proc"): - # Not the printer — a dry-run rehearsal against copied databases on a - # workstation. There is no daemon here to collide with. + if not os.path.isdir(proc): return None - for pid in os.listdir("/proc"): + for pid in os.listdir(proc): if not pid.isdigit(): continue try: - with open("/proc/%s/cmdline" % pid, "rb") as fh: + with open(os.path.join(proc, pid, "cmdline"), "rb") as fh: cmd = fh.read().decode("utf-8", "replace") except (IOError, OSError): continue @@ -165,6 +187,76 @@ def moonraker_running(): return None +def table_columns(db, table): + """{name: (notnull, has_default)} for one table, or {} if it is absent.""" + con = sqlite3.connect("file:%s?mode=ro" % db, uri=True) + try: + rows = con.execute("pragma table_info(%s)" % table).fetchall() + finally: + con.close() + # cid, name, type, notnull, dflt_value, pk + return dict((r[1], (bool(r[3]), r[4] is not None or bool(r[5]))) for r in rows) + + +def schema_problems(into_db, source_db): + """Everything about the two schemas that would make the write go wrong. + + Run during the DRY RUN, so drift is reported before anything is renamed or + written - not from inside the transaction, where the traceback lands in the + middle of a retirement with both daemons already stopped. + + COLUMNS is a hardcoded tuple checked once on one machine, and + `install_moonraker_nginx` runs `git checkout master; git pull` against the + Moonraker source - so job_history's schema is a MOVING TARGET under any + printer. Two failure shapes matter and only one of them is loud: + + - a column in COLUMNS that no longer exists raises inside the transaction. + Safe (it rolls back) but it aborts the retirement with a traceback. + - a column ADDED to the target that COLUMNS does not know about is worse: + the insert succeeds and every migrated row carries a NULL where + Moonraker's own reader expects a value. Nothing complains, ever. + + Note the comment on COLUMNS worries about column ORDER; that is not the + risk, because the insert names its columns. Existence and nullability are. + """ + problems = [] + target = table_columns(into_db, "job_history") + source = table_columns(source_db, "job_history") + if not target or not source: + return ["job_history table missing from %s" + % (into_db if not target else source_db)] + for name in COLUMNS: + for label, cols in (("target", target), ("source", source)): + if name not in cols: + problems.append( + "%s job_history has no '%s' column - this script is older " + "than the Moonraker it is writing to" % (label, name)) + for name, (notnull, has_default) in target.items(): + if name in COLUMNS or name == "job_id": + continue + if notnull and not has_default: + problems.append( + "target job_history has a NOT NULL column '%s' this script does " + "not write and that has no default" % name) + return problems + + +def dropped_columns(source_db): + """Source columns this script will not carry across. Reported, not fatal.""" + known = set(COLUMNS) | {"job_id"} + return sorted(c for c in table_columns(source_db, "job_history") if c not in known) + + +def instance_ids(db): + con = sqlite3.connect("file:%s?mode=ro" % db, uri=True) + try: + rows = con.execute( + "select distinct instance_id from job_history").fetchall() + finally: + con.close() + return sorted(r[0] for r in rows) + + def load_jobs(db): con = sqlite3.connect("file:%s?mode=ro" % db, uri=True) con.row_factory = sqlite3.Row @@ -174,12 +266,52 @@ def load_jobs(db): return rows, totals -def is_duplicate(row, targets): - for t in targets: - if row["filename"] == t["filename"] and \ - abs(row["start_time"] - t["start_time"]) <= TOLERANCE_S: - return t - return None +def match_pairs(source_rows, target_rows, eligible): + """Pair source rows to target rows ONE TO ONE, closest start_time first. + + Both properties matter and neither is obvious. + + ONE TO ONE, because otherwise N source rows all collapse onto the SAME + target row and every one of them but the first is called a duplicate and + silently dropped. That is not hypothetical: a cancelled print and its + immediate retry, same file, a minute apart, is routine, and if the target + daemon recorded only one of the pair - which happened twice on the reference + unit, and is half the reason this merge exists - both source rows land + inside the 120 s window of that single target row. + + CLOSEST FIRST rather than in table order, because a greedy pass in + start_time order gets the right COUNTS and the wrong ROWS. With a target + holding only the retry, source [failed, retry] pairs `failed` to the retry + (60 s apart, within tolerance) and then inserts `retry` as new - so the + target ends up with the retry twice and the failed print not at all. Sorting + every candidate pair by distance and assigning greedily from the closest + makes the exact match win its own row, which leaves the genuinely unmatched + row to be inserted. + + `eligible` is the set of source indices that are up for matching at all; + in_progress rows are excluded by the caller before we get here. + """ + candidates = [] + for si in eligible: + row = source_rows[si] + for ti, t in enumerate(target_rows): + if row["filename"] != t["filename"]: + continue + delta = abs(row["start_time"] - t["start_time"]) + if delta <= TOLERANCE_S: + candidates.append((delta, si, ti)) + # Sorted by (delta, si, ti): the tie-break on the indices keeps the result + # deterministic for two equidistant candidates rather than dependent on the + # sort's stability guarantees. + candidates.sort() + matched = {} + claimed = set() + for _, si, ti in candidates: + if si in matched or ti in claimed: + continue + matched[si] = ti + claimed.add(ti) + return matched def classify(source_rows, target_rows): @@ -190,13 +322,17 @@ def classify(source_rows, target_rows): NULL leaves a job that never completes. Filing it under "duplicate" would report a DROP as a no-op, which is the one thing a dry run must not do. """ + eligible = [i for i, row in enumerate(source_rows) + if row["status"] != "in_progress"] + matched = match_pairs(source_rows, target_rows, eligible) new, dupes, skipped = [], [], [] - for row in source_rows: + for index, row in enumerate(source_rows): if row["status"] == "in_progress": skipped.append(row) - continue - match = is_duplicate(row, target_rows) - (dupes if match else new).append((row, match)) + elif index in matched: + dupes.append((row, target_rows[matched[index]])) + else: + new.append((row, None)) return new, dupes, skipped @@ -223,6 +359,35 @@ def merge_totals(target_totals, source_totals): return merged +def renumber_by_start_time(con): + """Make job_id order match start_time order, inside the caller's transaction. + + Moonraker pages history with `ORDER BY job_id`, so after appending older + prints the id order IS the display order and it is wrong. See the SAFETY + note in the header for why this is worth the id churn. + + Done as an offset pass rather than in place: job_id is `INTEGER PRIMARY KEY + ASC`, so assigning 1..N directly would collide with rows that still hold + those ids. Shifting every row above the current maximum first makes the + second pass collision-free without needing a temp table. + + Ordered by (start_time, job_id) so rows sharing a timestamp keep their + existing relative order rather than being permuted arbitrarily. + """ + rows = con.execute( + "select job_id from job_history order by start_time, job_id").fetchall() + if not rows: + return + offset = con.execute( + "select coalesce(max(job_id), 0) from job_history").fetchone()[0] + for position, row in enumerate(rows, start=1): + con.execute("update job_history set job_id = ? where job_id = ?", + (offset + position, row[0])) + for position in range(1, len(rows) + 1): + con.execute("update job_history set job_id = ? where job_id = ?", + (position, offset + position)) + + def main(): ap = argparse.ArgumentParser( description="fold one K1C 2025 print history into the other") @@ -237,6 +402,10 @@ def main(): ap.add_argument("--force", action="store_true", help="write even with a daemon alive — it will then overwrite " "the merged job_totals from its stale in-memory copy") + ap.add_argument("--allow-instance-mismatch", action="store_true", + help="copy rows even when the two databases scope their " + "history to different instance_ids — the copies will " + "not be visible to the daemon reading them") args = ap.parse_args() default_source, default_into = DIRECTIONS[args.direction] @@ -247,6 +416,16 @@ def main(): if not os.path.exists(path): sys.exit("missing database: %s" % path) + # Schema first, before anything is read or reported. A mismatch here means + # the write would fail (or, worse, silently half-succeed), and the only safe + # moment to say so is before the caller starts renaming init scripts. + problems = schema_problems(into_db, source_db) + if problems: + sys.exit("refusing: schema mismatch\n " + "\n ".join(problems)) + dropped = dropped_columns(source_db) + if dropped: + print("note: source columns not copied: %s" % ", ".join(dropped)) + target_rows, target_totals = load_jobs(into_db) source_rows, source_totals = load_jobs(source_db) @@ -260,9 +439,28 @@ def listing(rows): print(" %s %-44s %s" % (when(row["start_time"]), (row["filename"] or "")[:44], row["status"])) + # Moonraker scopes every history query by instance_id (its own list handler + # filters on a bare "default"), so rows carrying a different one insert + # successfully and are then invisible in Fluidd and in the screen's count - + # a merge that reports success and shows nothing. Report both sets always, + # and refuse when they cannot see each other. + target_instances = instance_ids(into_db) + source_instances = instance_ids(source_db) print("direction: %s" % args.direction) - print("target %s: %d jobs" % (into_db, len(target_rows))) - print("source %s: %d jobs" % (source_db, len(source_rows))) + print("target %s: %d jobs, instance_id %s" + % (into_db, len(target_rows), target_instances or ["(empty)"])) + print("source %s: %d jobs, instance_id %s" + % (source_db, len(source_rows), source_instances or ["(empty)"])) + if target_instances and source_instances and \ + not set(target_instances) & set(source_instances): + if not args.allow_instance_mismatch: + sys.exit( + "refusing: the two databases scope their history to different\n" + "instance_ids (%s vs %s). Copied rows would be invisible to the\n" + "daemon that reads them, and the merge would report success.\n" + "Re-run with --allow-instance-mismatch if that is really wanted." + % (target_instances, source_instances)) + print("warning: instance_id mismatch, copying anyway (--allow-instance-mismatch)") print("duplicate (skipped): %d" % len(dupes)) print("in_progress (NOT copied): %d" % len(skipped)) listing(skipped) @@ -287,8 +485,14 @@ def listing(rows): backup = "%s.bak-merge-%s" % (into_db, time.strftime("%Y%m%d_%H%M%S")) shutil.copy2(into_db, backup) + # Name the daemon that actually owns the file being replaced. The forward + # merge writes Moonraker's database and the reverse one writes nexusp's, so + # a fixed "stop Moonraker" tells the user to stop the wrong daemon on the + # restore path - and this line is the only rollback instruction that appears + # anywhere. + owner = "nexusp" if into_db == NEXUSP_DB else "Moonraker" print("\nbackup: %s" % backup) - print("rollback: stop Moonraker, cp %s %s, restart it" % (backup, into_db)) + print("rollback: stop %s, cp %s %s, restart it" % (owner, backup, into_db)) con = sqlite3.connect(into_db) con.row_factory = sqlite3.Row @@ -304,6 +508,7 @@ def listing(rows): "insert or replace into job_totals " "(provider, field, maximum, total, instance_id) values (?,?,?,?,?)", (provider, field, t["maximum"], t["total"], inst)) + renumber_by_start_time(con) total = con.execute("select count(*) from job_history").fetchone()[0] print("merged: %d jobs in %s" % (total, into_db)) finally: diff --git a/files/moonraker/creality-compat/test_creality_compat.py b/files/moonraker/creality-compat/test_creality_compat.py index 92d9bb4..ec96d46 100644 --- a/files/moonraker/creality-compat/test_creality_compat.py +++ b/files/moonraker/creality-compat/test_creality_compat.py @@ -430,6 +430,18 @@ def test_keyword_narrows_the_count_not_just_the_page(): assert call(shim, keyword="a")["count"] == 2 +def test_a_keyword_does_not_hide_subdirectories(): + """Files narrow, folders do not. A search that hides the folder you were + about to open is the same "it looks deleted" failure this shim exists to + avoid, and the golden captures do not cover keyword-with-directories — so + this is the chosen reading, not a measured one. Pinned either way.""" + shim, _, _ = build(dirs=[entry("models", True), entry("spares", True)], + files=[entry("cube.gcode"), entry("other.gcode")]) + result = call(shim, keyword="cube") + assert names(result) == [("d", "models"), ("d", "spares"), ("f", "cube.gcode")] + assert result["count"] == 3 + + def test_count_is_the_total_before_paging(): shim, _, _ = build(files=[entry("%02d.gcode" % i) for i in range(10)]) result = call(shim, start=0, limit=3) @@ -619,24 +631,45 @@ def test_a_directory_with_no_thumbs_is_not_an_error(tmp_path): assert call(shim)["items"][0].get("thumbnails") in (None, []) -def test_only_the_page_is_decorated(tmp_path): +def test_only_the_page_is_decorated(tmp_path, fake_pil): """Thumbnail work happens AFTER the slice. Sorting reads filename/modified/ size and never thumbnails, so nothing above needs it — and doing it first meant a 91-file directory did the work for all 91 to serve 20 rows, on the - event loop that also serves klippy and Fluidd, during a print.""" - root = thumbs_dir(tmp_path, "a-195x195.png", "b-195x195.png") + event loop that also serves klippy and Fluidd, during a print. + + Asserted on the WORK DONE, not on the page contents. An earlier version of + this test checked only that the page held one file, which is true whether + the merge runs before or after the slice — moving the decoration back before + the slice left the whole suite green. + """ + root = thumbs_dir(tmp_path, "a-300x300.png", "b-300x300.png") shim, _, _ = build(disk_root=root, files=[entry("a.gcode"), entry("b.gcode")]) result = call(shim, start=0, limit=1) assert result["count"] == 2 assert result["items"][0]["filename"] == "a.gcode" assert result["items"][0]["thumbnails"] + # b.gcode is off the page, so nothing may have been rendered for it. + assert {os.path.basename(p) for p, _ in fake_pil.calls} == {"a-300x300.png"} -def test_an_empty_page_does_no_thumbnail_work(tmp_path): - root = thumbs_dir(tmp_path, "a-195x195.png") +def test_an_empty_page_does_no_thumbnail_work(tmp_path, fake_pil): + root = thumbs_dir(tmp_path, "a-300x300.png") shim, _, _ = build(disk_root=root, files=[entry("a.gcode")]) assert call(shim, start=0, limit=0)["items"] == [] + assert fake_pil.calls == [] + + +def test_the_response_carries_every_key_nexusp_returned(tmp_path): + """Pinned against the ten golden captures, whose result keys are exactly + {count, disk_usage, items, root_info}. An earlier version dropped + disk_usage, which the screen reads its free-space figure from — a blank + readout with no error, which is the failure class this option exists to + remove.""" + shim, _, _ = build(files=[entry("a.gcode")]) + result = call(shim) + assert sorted(result) == ["count", "disk_usage", "items", "root_info"] + assert result["disk_usage"] == {"total": 1, "used": 0, "free": 1} # -------------------------------------------------------------------------- @@ -820,61 +853,33 @@ def test_a_missing_pillow_still_lists_every_thumbnail_on_disk(tmp_path): # -------------------------------------------------------------------------- -# `.thumbs` is kept out of file_manager's inotify watch +# Writes are bounded, and nothing is reserved # -------------------------------------------------------------------------- -def test_existing_thumb_dirs_are_reserved_at_load(tmp_path, fake_pil): - """Writing a PNG into a watched directory fires `notify_filelist_changed`, - and file_manager has no dot-directory filter — so generating thumbnails - inside a listing would broadcast a phantom `create_file` for every PNG to - Fluidd and to the screen. Reserving has to happen before file_manager's - initial scan, which runs in its `component_init`, after every component is - constructed.""" - root = thumbs_dir(tmp_path, "Cube-195x195.png") - sub = tmp_path / "sub" / ".thumbs" - sub.mkdir(parents=True) - _, fm, _ = build(disk_root=root) - reserved = {path for path, _ in fm.reserved.values()} - assert reserved == {os.path.join(root, ".thumbs"), str(sub)} - - -def test_reserved_thumb_dirs_stay_readable(tmp_path, fake_pil): - """Read access is left on, or the thumbnails stop being served over HTTP - and every tile goes blank — the exact failure this is meant to fix.""" - root = thumbs_dir(tmp_path, "Cube-195x195.png") - _, fm, _ = build(disk_root=root) - assert all(read_access for _, read_access in fm.reserved.values()) +def test_renders_are_capped_per_request(tmp_path, fake_pil): + """Each generated PNG makes Moonraker broadcast a phantom `create_file` — + file_manager watches every directory under gcodes and has no dot-directory + filter. The cap bounds the burst one large page can produce; the files past + it get their thumbnails on a later listing. + """ + names = ["f%02d-300x300.png" % i for i in range(20)] + root = thumbs_dir(tmp_path, *names) + shim, _, _ = build(disk_root=root, + files=[entry("f%02d.gcode" % i) for i in range(20)]) + call(shim, limit=100) + assert len(fake_pil.calls) == cc.MAX_RENDERS_PER_REQUEST -def test_a_new_thumb_dir_is_reserved_before_it_is_created(tmp_path, fake_pil): - """The directory-create inotify event checks the reserved list, so the - reservation only works if it lands first. Here `.thumbs` does not exist at - load — the source is a slicer-embedded thumbnail — so the component has to - reserve it on the way to creating it.""" - root = str(tmp_path) - shim, fm, _ = build(disk_root=root, files=[ - dict(entry("Cube.gcode"), thumbnails=[ - {"width": 300, "height": 300, "size": 9, - "relative_path": ".thumbs/Cube-300x300.png"}])]) - assert fm.reserved == {} +def test_the_component_reserves_nothing(tmp_path, fake_pil): + """A reserved path is not just a notification filter: check_reserved_path + with need_write=True guards delete/move/upload, so reserving `.thumbs` would + make every thumbnail permanently undeletable through Fluidd, with no removal + API — on a printer where clearing them is exactly what people do for space. + The phantom notifications are the cheaper problem. + """ + root = thumbs_dir(tmp_path, "Cube-300x300.png") + shim, fm, _ = build(disk_root=root, files=[entry("Cube.gcode")]) call(shim) - assert os.path.join(root, ".thumbs") in {p for p, _ in fm.reserved.values()} - assert os.path.isdir(os.path.join(root, ".thumbs")) - - -def test_nothing_is_reserved_when_generation_is_off(tmp_path, fake_pil): - """No writes, no phantom notifications, no reason to hide `.thumbs` from - Fluidd's file list.""" - root = thumbs_dir(tmp_path, "Cube-195x195.png") - _, fm, _ = build(disk_root=root, generate_thumbnails=False) - assert fm.reserved == {} - - -def test_nothing_is_reserved_when_pillow_is_missing(tmp_path): - """Same reasoning: without Pillow the component never writes a PNG, so - there is nothing to keep out of the watch.""" - root = thumbs_dir(tmp_path, "Cube-195x195.png") - _, fm, _ = build(disk_root=root) assert fm.reserved == {} diff --git a/files/moonraker/creality-compat/test_merge_job_history.py b/files/moonraker/creality-compat/test_merge_job_history.py index b414747..81945e3 100644 --- a/files/moonraker/creality-compat/test_merge_job_history.py +++ b/files/moonraker/creality-compat/test_merge_job_history.py @@ -53,6 +53,10 @@ import merge_job_history as mjh # noqa: E402 +# Captured before the autouse fixture below stubs it out, so the handful of +# tests that are ABOUT the guard can still reach the real implementation. +REAL_MOONRAKER_RUNNING = mjh.moonraker_running + # Verbatim from the reference unit. moonraker declares metadata/auxiliary_data # as `pyjson` and nexusp as TEXT; sqlite type names are advisory and the script # connects without detect_types, so one DDL serves both here. @@ -155,6 +159,19 @@ def run_argv(monkeypatch, capsys, *flags): return capsys.readouterr().out +@pytest.fixture(autouse=True) +def no_daemon_by_default(monkeypatch): + """These tests are about the merge, not about the host's process table. + + Without this every `--apply` test reads the REAL /proc. They pass on a Mac + only because /proc does not exist there; on Linux — including the printer, + where a user is most likely to run them — a live Moonraker makes main() + SystemExit and they all fail for a reason unrelated to what they assert. + The two tests that are about the guard opt back in explicitly. + """ + monkeypatch.setattr(mjh, "moonraker_running", lambda *a, **k: None) + + # -------------------------------------------------------------------------- # The dedupe window # -------------------------------------------------------------------------- @@ -195,6 +212,35 @@ def test_a_failed_print_and_its_retry_both_survive(): assert (len(new), len(dupes)) == (1, 1) +def test_two_source_rows_never_collapse_onto_one_target_row(): + """THE DATA LOSS CASE. Matching must be one to one. + + A cancelled print and its retry a minute apart, where the target daemon + recorded only ONE of the pair — which is not exotic, it is half the reason + this merge exists (the reference unit's Moonraker missed two prints the + screen saw). Both source rows sit inside the 120 s window of that single + target row. Without claim tracking both are called duplicates and the print + that exists nowhere else is dropped, silently, with the dry run reporting + 'to insert: 0'. + """ + target = [job(T0 + 60)] + new, dupes, _ = mjh.classify([job(T0), job(T0 + 60)], target) + assert (len(new), len(dupes)) == (1, 1) + # And it must be the RIGHT row. Matching greedily in table order gets these + # counts while pairing the failed print to the retry's row, which inserts + # the retry a second time and loses the failed print anyway. + assert new[0][0]["start_time"] == T0 + assert dupes[0][0]["start_time"] == T0 + 60 + + +def test_the_nearest_start_time_wins_a_contested_match(): + """Closest first, not table order — an exact match must win its own row.""" + target = [job(T0 + 100), job(T0 + 5)] + new, dupes, _ = mjh.classify([job(T0)], target) + assert len(dupes) == 1 + assert dupes[0][1]["start_time"] == T0 + 5 + + def test_different_files_at_the_same_instant_are_distinct(): target = [job(T0, filename="a.gcode")] new, dupes, _ = mjh.classify([job(T0, filename="b.gcode")], target) @@ -291,7 +337,7 @@ def test_totals_land_in_the_database_as_a_replace(monkeypatch, capsys, tmp_path) # -------------------------------------------------------------------------- def test_a_live_daemon_blocks_and_force_overrides(monkeypatch, capsys, tmp_path): - monkeypatch.setattr(mjh, "moonraker_running", lambda: "python moonraker.py") + monkeypatch.setattr(mjh, "moonraker_running", lambda *a, **k: "python moonraker.py") into = make_db(tmp_path / "into.db", [job(T0)]) source = make_db(tmp_path / "src.db", [job(T0 + 9999)]) with pytest.raises(SystemExit): @@ -302,11 +348,58 @@ def test_a_live_daemon_blocks_and_force_overrides(monkeypatch, capsys, tmp_path) assert len(rows_of(into)) == 2 -def test_no_proc_reads_as_no_daemon(monkeypatch): +def test_no_proc_reads_as_no_daemon(tmp_path): """The workstation rehearsal path: /proc does not exist on a Mac, and the guard must degrade to "nothing running here" rather than crash.""" - monkeypatch.setattr(os.path, "isdir", lambda p: False) - assert mjh.moonraker_running() is None + assert REAL_MOONRAKER_RUNNING(str(tmp_path / "no-such-proc")) is None + + +# The guard's own matching logic. Every other test stubs moonraker_running out, +# so without these the substring match is never executed against a realistic +# cmdline — and this is the only thing standing between a live daemon and an +# irreversible write to a user's print history. + +def fake_proc(tmp_path, **pids): + proc = tmp_path / "proc" + proc.mkdir() + for pid, cmd in pids.items(): + (proc / pid).mkdir() + (proc / pid / "cmdline").write_bytes(cmd) + (proc / "cpuinfo").write_text("not a pid") + return str(proc) + + +def test_a_live_moonraker_is_detected(tmp_path): + """The exact shape S56moonraker_service produces: the venv interpreter with + moonraker.py as an argument, which is what busybox ps truncates away.""" + proc = fake_proc(tmp_path, **{"42": ( + b"/usr/data/moonraker/moonraker-env/bin/python3\x00" + b"/usr/data/moonraker/moonraker/moonraker/moonraker.py\x00" + b"-d\x00/usr/data/printer_data\x00")}) + assert "moonraker.py" in REAL_MOONRAKER_RUNNING(proc) + + +def test_a_live_nexusp_is_detected(tmp_path): + proc = fake_proc(tmp_path, **{"77": b"/usr/bin/nexusp\x00-d\x00/usr/data/printer_data\x00"}) + assert "nexusp" in REAL_MOONRAKER_RUNNING(proc) + + +def test_an_unrelated_process_is_not_mistaken_for_a_daemon(tmp_path): + """Including this script itself — it has 'merge_job_history.py' in its + cmdline, not 'moonraker.py', and must not block its own run.""" + proc = fake_proc(tmp_path, **{ + "7": b"/usr/bin/klipper\x00", + "9": (b"python3\x00/usr/data/helper-script/files/moonraker/" + b"creality-compat/merge_job_history.py\x00--apply\x00")}) + assert REAL_MOONRAKER_RUNNING(proc) is None + + +def test_an_unreadable_cmdline_is_skipped_not_fatal(tmp_path): + """/proc entries race with process exit; a vanished pid must not abort the + scan and let a different live daemon through unnoticed.""" + proc = fake_proc(tmp_path, **{"11": b"/usr/bin/nexusp\x00"}) + os.mkdir(os.path.join(proc, "12")) # a pid dir with no cmdline at all + assert "nexusp" in REAL_MOONRAKER_RUNNING(proc) def test_a_missing_database_exits_before_touching_anything(monkeypatch, capsys, @@ -317,6 +410,119 @@ def test_a_missing_database_exits_before_touching_anything(monkeypatch, capsys, assert glob.glob(into + ".bak-merge-*") == [] +# -------------------------------------------------------------------------- +# Schema drift and instance scoping — checked in the DRY RUN, before the +# caller has renamed anything +# -------------------------------------------------------------------------- + +def test_a_missing_column_is_refused_before_any_write(monkeypatch, capsys, tmp_path): + """`install_moonraker_nginx` runs `git checkout master; git pull` on the + Moonraker source, so job_history's schema is a moving target under any + printer. Discovering that inside the transaction means a traceback in the + middle of a retirement with both daemons already stopped.""" + into = str(tmp_path / "into.db") + con = sqlite3.connect(into) + con.execute(JOB_HISTORY_DDL.replace(" filament_used REAL NOT NULL,\n", "")) + con.execute(JOB_TOTALS_DDL) + con.commit() + con.close() + source = make_db(tmp_path / "src.db", [job(T0)]) + with pytest.raises(SystemExit) as exc: + run_main(monkeypatch, capsys, into, source) + assert "filament_used" in str(exc.value) + assert glob.glob(into + ".bak-merge-*") == [] + + +def test_an_added_not_null_column_is_refused(monkeypatch, capsys, tmp_path): + """The silent one. A column the script does not write, NOT NULL with no + default, would make every migrated row fail — or, if it were nullable, + succeed while carrying a NULL the daemon does not expect.""" + into = str(tmp_path / "into.db") + con = sqlite3.connect(into) + con.execute(JOB_HISTORY_DDL.replace( + " instance_id TEXT NOT NULL\n", + " instance_id TEXT NOT NULL,\n new_field TEXT NOT NULL\n")) + con.execute(JOB_TOTALS_DDL) + con.commit() + con.close() + source = make_db(tmp_path / "src.db", [job(T0)]) + with pytest.raises(SystemExit) as exc: + run_main(monkeypatch, capsys, into, source) + assert "new_field" in str(exc.value) + + +def test_an_added_nullable_column_is_allowed_and_reported(monkeypatch, capsys, + tmp_path): + """Nullable additions are survivable, so they must not block a retirement — + but the source columns being dropped are worth naming.""" + into = str(tmp_path / "into.db") + con = sqlite3.connect(into) + con.execute(JOB_HISTORY_DDL.replace( + " instance_id TEXT NOT NULL\n", + " instance_id TEXT NOT NULL,\n new_field TEXT\n")) + con.execute(JOB_TOTALS_DDL) + con.commit() + con.close() + source = make_db(tmp_path / "src.db", [job(T0)]) + out = run_main(monkeypatch, capsys, into, source, "--apply") + assert len(rows_of(into)) == 1 + + +def test_a_source_only_column_is_reported_as_dropped(monkeypatch, capsys, tmp_path): + into = make_db(tmp_path / "into.db", [job(T0)]) + source = str(tmp_path / "src.db") + con = sqlite3.connect(source) + con.execute(JOB_HISTORY_DDL.replace( + " instance_id TEXT NOT NULL\n", + " instance_id TEXT NOT NULL,\n creality_extra TEXT\n")) + con.execute(JOB_TOTALS_DDL) + con.commit() + con.close() + out = run_main(monkeypatch, capsys, into, source) + assert "creality_extra" in out + + +def test_disjoint_instance_ids_are_refused(monkeypatch, capsys, tmp_path): + """Moonraker scopes every history query by instance_id. Rows carrying one + the reader does not filter on insert fine and are then invisible — a merge + that prints success and shows nothing, which is the exact failure mode this + whole option exists to remove.""" + into = make_db(tmp_path / "into.db", [job(T0, instance_id="default")]) + source = make_db(tmp_path / "src.db", + [job(T0 - 86400, instance_id="creality")]) + with pytest.raises(SystemExit) as exc: + run_main(monkeypatch, capsys, into, source, "--apply") + assert "instance_id" in str(exc.value) + assert len(rows_of(into)) == 1 + assert glob.glob(into + ".bak-merge-*") == [] + + +def test_an_instance_mismatch_can_be_overridden_explicitly(monkeypatch, capsys, + tmp_path): + into = make_db(tmp_path / "into.db", [job(T0, instance_id="default")]) + source = make_db(tmp_path / "src.db", + [job(T0 - 86400, instance_id="creality")]) + run_main(monkeypatch, capsys, into, source, "--apply", + "--allow-instance-mismatch") + assert len(rows_of(into)) == 2 + + +def test_an_empty_target_does_not_trip_the_instance_check(monkeypatch, capsys, + tmp_path): + """A fresh install has no rows and therefore no instance_id to compare.""" + into = make_db(tmp_path / "into.db", []) + source = make_db(tmp_path / "src.db", [job(T0, instance_id="creality")]) + run_main(monkeypatch, capsys, into, source, "--apply") + assert len(rows_of(into)) == 1 + + +def test_the_dry_run_names_both_instance_ids(monkeypatch, capsys, tmp_path): + into = make_db(tmp_path / "into.db", [job(T0)]) + source = make_db(tmp_path / "src.db", [job(T0 - 86400)]) + out = run_main(monkeypatch, capsys, into, source) + assert out.count("instance_id") >= 2 + + # -------------------------------------------------------------------------- # Direction # -------------------------------------------------------------------------- @@ -440,10 +646,17 @@ def test_a_second_apply_inserts_nothing(monkeypatch, capsys, tmp_path): assert rows_of(into) == first -def test_existing_job_ids_are_never_rewritten(monkeypatch, capsys, tmp_path): - """Renumbering would rewrite the identity of rows a client may already - hold — a Fluidd tab left open across the merge would then delete by an id - that names a different print. Inserted rows continue after the target's. +def test_job_ids_end_up_in_start_time_order(monkeypatch, capsys, tmp_path): + """THE ORDERING PROPERTY. Moonraker pages history with `ORDER BY job_id` + (history.py, order defaults to desc) and has no ORDER BY start_time at all, + so after a merge the id order IS the display order. + + Appending without renumbering gave the recovered prints — the OLDEST on the + machine, which is the whole point of the merge — the HIGHEST ids, so they + came back as the most recent jobs in Fluidd and on the screen while + server.history.count reported the right total. A correct scrollbar over a + wrongly ordered list is the plausible-wrong-answer failure this option + exists to remove. """ into = make_db(tmp_path / "into.db", [job(T0, job_id=41, filename="a.gcode"), @@ -451,9 +664,42 @@ def test_existing_job_ids_are_never_rewritten(monkeypatch, capsys, tmp_path): source = make_db(tmp_path / "src.db", [job(T0 - 86400, filename="ancient.gcode")]) run_main(monkeypatch, capsys, into, source, "--apply") - by_name = {r["filename"]: r["job_id"] for r in rows_of(into)} - assert by_name["a.gcode"] == 41 - assert by_name["b.gcode"] == 42 - # The oldest print by time gets the HIGHEST id. That inversion is the - # deliberate trade: ids stay stable, ordering is start_time's job. - assert by_name["ancient.gcode"] == 43 + by_id = [r["filename"] for r in rows_of(into, order="job_id")] + assert by_id == ["ancient.gcode", "a.gcode", "b.gcode"] + ids = [r["job_id"] for r in rows_of(into, order="job_id")] + assert ids == sorted(ids) and len(set(ids)) == len(ids) + + +def test_renumbering_is_a_dense_sequence_from_one(monkeypatch, capsys, tmp_path): + """The offset pass must not leave gaps or strand rows at the shifted ids.""" + into = make_db(tmp_path / "into.db", + [job(T0 + i * 1000, job_id=100 + i, filename="f%d.gcode" % i) + for i in range(3)]) + source = make_db(tmp_path / "src.db", + [job(T0 - 86400, filename="old.gcode")]) + run_main(monkeypatch, capsys, into, source, "--apply") + assert [r["job_id"] for r in rows_of(into, order="job_id")] == [1, 2, 3, 4] + + +def test_rows_sharing_a_start_time_keep_their_relative_order(monkeypatch, capsys, + tmp_path): + """Ties break on the existing job_id, so a merge does not permute rows it + had no reason to touch.""" + into = make_db(tmp_path / "into.db", + [job(T0, job_id=5, filename="first.gcode"), + job(T0, job_id=6, filename="second.gcode")]) + source = make_db(tmp_path / "src.db", [job(T0 - 86400, filename="old.gcode")]) + run_main(monkeypatch, capsys, into, source, "--apply") + by_id = [r["filename"] for r in rows_of(into, order="job_id")] + assert by_id == ["old.gcode", "first.gcode", "second.gcode"] + + +def test_renumbering_survives_a_second_run(monkeypatch, capsys, tmp_path): + """Idempotency still holds: the second pass inserts nothing and the ids it + assigns are the ones already there.""" + into = make_db(tmp_path / "into.db", [job(T0, filename="a.gcode")]) + source = make_db(tmp_path / "src.db", [job(T0 - 86400, filename="old.gcode")]) + run_main(monkeypatch, capsys, into, source, "--apply") + first = rows_of(into, order="job_id") + run_main(monkeypatch, capsys, into, source, "--apply") + assert rows_of(into, order="job_id") == first diff --git a/scripts/menu/K1_2025/info_menu_K1C_2025.sh b/scripts/menu/K1_2025/info_menu_K1C_2025.sh index f453397..54c5d91 100755 --- a/scripts/menu/K1_2025/info_menu_K1C_2025.sh +++ b/scripts/menu/K1_2025/info_menu_K1C_2025.sh @@ -68,7 +68,13 @@ function check_creality_services_k1_2025() { # race for port 7125 to Moonraker at every boot. Retire Nexusp Backend offers # the repair. function check_nexusp_retired_k1_2025() { - if nexusp_resurrected; then + if nexusp_absent; then + # This firmware ships no nexusp in either form, so there is nothing to + # retire. A red cross here would read as an unfinished action on a printer + # where the action does not apply - the Customize menu already checks + # nexusp_absent first for the same reason. + echo -e "${cyan}-" + elif nexusp_resurrected; then echo -e "${yellow}~" elif nexusp_retired; then echo -e "${green}✓" diff --git a/scripts/moonraker_nginx.sh b/scripts/moonraker_nginx.sh index f66a55c..b8afdc0 100755 --- a/scripts/moonraker_nginx.sh +++ b/scripts/moonraker_nginx.sh @@ -56,8 +56,6 @@ function moonraker_3v3_message(){ } function configure_moonraker_nginx_k1_2025(){ - local nginx_conf - # Moonraker moves to 7126 because Creality's nexusp squats on 7125 - UNLESS # Retire Nexusp Backend has already turned nexusp off, in which case 7125 is # ours and the touchscreen is pointed at it. install_moonraker_nginx rm -f's @@ -67,21 +65,20 @@ function configure_moonraker_nginx_k1_2025(){ # touchscreen died, and there was no error anywhere to connect it to. if nexusp_retired && ! nexusp_present; then echo -e "Info: Nexusp is retired, keeping Moonraker on port 7125..." - nexusp_reapply_retired_config + # Guarded: these helpers now report write failures instead of letting + # errexit escape, and an unguarded call would abort the whole install. + if ! nexusp_reapply_retired_config; then + echo -e "${yellow}Warning: could not re-apply the retired configuration.${white}" + echo -e "${yellow}Nothing is answering port 7125 - run Retire Nexusp Backend${white}" + echo -e "${yellow}again to finish, or Restore Nexusp Backend to go back.${white}" + fi return fi - if [ -f "$MOONRAKER_CFG" ]; then - echo -e "Info: Setting Moonraker port to 7126..." - sed -i 's/^port:[[:space:]]*7125$/port: 7126/' "$MOONRAKER_CFG" - fi - - for nginx_conf in "$NGINX_FOLDER"/nginx/nginx.conf /etc/nginx/nginx.conf; do - if [ -f "$nginx_conf" ]; then - echo -e "Info: Pointing Nginx Moonraker upstream to port 7126 in $nginx_conf..." - sed -i 's/server 127\.0\.0\.1:7125;/server 127.0.0.1:7126;/' "$nginx_conf" - fi - done + # One implementation of the swap, not two. Both directions have to stay exact + # inverses of each other, and a second hand-inlined copy of the same two seds + # is how they stop being. + nexusp_set_moonraker_port 7126 7125 } function install_moonraker_nginx(){ @@ -145,6 +142,19 @@ function remove_moonraker_nginx(){ case "${yn}" in Y|y) echo -e "${white}" + # On a retired box the touchscreen's ONLY backend is this Moonraker. + # Removing it while nexusp stays disabled leaves nothing bound to :7125 + # - not now and not after any reboot - so the screen dies permanently + # with nothing connecting it to this menu entry. install_moonraker_nginx + # got this guard; the adjacent removal needs it just as much. + if [ "$model" = "K1_2025" ] && nexusp_retired && ! nexusp_present; then + error_msg "Nexusp Backend is retired, so this Moonraker is the touchscreen's only backend!" + echo -e " ${darkred}Removing it now would leave nothing answering port 7125 and${white}" + echo -e " ${darkred}the touchscreen dead permanently.${white}" + echo -e " ${cyan}Run Restore Nexusp Backend first, then remove.${white}" + echo + return + fi echo -e "Info: Stopping Moonraker and Nginx services..." stop_moonraker stop_nginx diff --git a/scripts/retire_nexusp.sh b/scripts/retire_nexusp.sh index 67a76c6..ac2db04 100644 --- a/scripts/retire_nexusp.sh +++ b/scripts/retire_nexusp.sh @@ -107,21 +107,22 @@ function nexusp_disabled_path() { echo "$(dirname "$1")/disabled.$(basename "$1")" } -# Both the S and CS prefixes: the init script name varies by firmware, and -# tools.sh and tools_menu_K1C_2025.sh already probe for both forms elsewhere. -function nexusp_service_files() { - echo "$NEXUSP_SERVICE $NEXUSP_SERVICE_LEGACY" -} - # The enabled init script, if there is one. Empty otherwise. # +# Both the S and CS prefixes are checked: the init script name varies by +# firmware, and tools.sh and tools_menu_K1C_2025.sh already probe for both forms +# elsewhere. The two paths are iterated as quoted variables rather than round +# tripped through an unquoted `$(...)` - these values are executed and `mv`d as +# root, so they must not be exposed to word splitting or glob expansion by a +# future edit to INITD_FOLDER. +# # The trailing `return 0` is load-bearing: without it a loop that finds nothing # returns the last failed `[ -f ]`, and `svc="$(nexusp_enabled_service)"` would # then abort the whole helper under the `set -e` helper.sh applies globally. # Absence is an answer here, not an error. function nexusp_enabled_service() { local svc - for svc in $(nexusp_service_files); do + for svc in "$NEXUSP_SERVICE" "$NEXUSP_SERVICE_LEGACY"; do if [ -f "$svc" ]; then echo "$svc" return 0 @@ -133,7 +134,7 @@ function nexusp_enabled_service() { # The disabled.* init script, if there is one. Empty otherwise. function nexusp_disabled_service() { local svc disabled_svc - for svc in $(nexusp_service_files); do + for svc in "$NEXUSP_SERVICE" "$NEXUSP_SERVICE_LEGACY"; do disabled_svc="$(nexusp_disabled_path "$svc")" if [ -f "$disabled_svc" ]; then echo "$disabled_svc" @@ -210,17 +211,83 @@ function nexusp_pillow_installed() { return $rc } +# Did Moonraker report the compat component as FAILED to load? +# +# This is the difference between "the port answers" and "the touchscreen works". +# Moonraker loads config-declared components with +# `load_component(config, section, None)`, and that call catches every exception, +# logs it, appends the name to `failed_components` and RETURNS - the server keeps +# serving. So a broken component (a Moonraker rename tripping the load-time +# hasattr guard, a bad import) leaves /server/info answering 200 while the file +# browser is dead, and the reason is only in moonraker.log. /server/info exposes +# both lists, so ask it directly. +# +# jq is not available on this path (see disable_creality_services.sh), so the +# body is trimmed to the failed_components array first - otherwise the component +# name appearing in the healthy `components` list would match too. +function nexusp_compat_load_failed() { + local failed + failed="$(echo "$1" | sed 's/.*"failed_components"//' | sed 's/\].*//')" + case "$failed" in + *creality_compat*) + return 0;; + *) + return 1;; + esac +} + +# Poll until Moonraker answers on :7125 AND has actually loaded the component. +# +# Polling, not a single probe: start_moonraker sleeps one second, and Moonraker's +# Python 3.8 startup on this board routinely takes longer than that. A one-shot +# check turns a slow-but-successful swap into an automatic rollback of a swap +# that worked, which is a worse outcome than the failure it is trying to catch. +function nexusp_verify_retired() { + local attempt body + attempt=0 + while [ "$attempt" -lt 30 ]; do + body="$("$CURL" -s -m 5 http://127.0.0.1:7125/server/info 2>/dev/null)" + if nexusp_compat_load_failed "$body"; then + error_msg "Moonraker started but refused to load the compat component!" + echo -e " ${darkred}The touchscreen's file browser would not work. See${white}" + echo -e " ${darkred}/usr/data/printer_data/logs/moonraker.log for the reason.${white}" + return 1 + fi + case "$body" in + *creality_compat*) + return 0;; + esac + attempt=$((attempt + 1)) + sleep 2 + done + error_msg "Nothing usable answered on port 7125 after 60 seconds!" + return 1 +} + # The reload the init script's own verb cannot be trusted to do. See the header. +# Returns non-zero when NO candidate config could be reloaded. That matters: +# nginx is the only thing the browser talks to, and a failed reload leaves it +# proxying the OLD upstream - the now-dead :7126 - so Fluidd 502s while +# Moonraker itself is perfectly healthy on :7125. Verifying the daemon directly +# cannot see that, because it bypasses nginx entirely. function nexusp_reload_nginx() { - local conf + local conf reloaded + reloaded="" echo -e "Info: Reloading Nginx..." set +e for conf in "$NGINX_CONF_FILE" /etc/nginx/nginx.conf; do if [ -f "$conf" ] && [ -x "$NGINX_BIN" ]; then - "$NGINX_BIN" -c "$conf" -s reload > /dev/null 2>&1 && break + if "$NGINX_BIN" -c "$conf" -s reload > /dev/null 2>&1; then + reloaded="1" + break + fi fi done set -e + if [ -z "$reloaded" ]; then + return 1 + fi + return 0 } function nexusp_stop_service() { @@ -256,24 +323,41 @@ function nexusp_set_moonraker_port() { local want="$1" other="$2" nginx_conf if [ -f "$MOONRAKER_CFG" ]; then echo -e "Info: Setting Moonraker port to ${want}..." - sed -i "s/^port:[[:space:]]*${other}\$/port: ${want}/" "$MOONRAKER_CFG" + # Guarded for the same reason the component install is: an unguarded + # `sed -i` failure on a full or read-only /usr/data would exit the helper + # here, halfway through a swap, with no rollback. + if ! sed -i "s/^port:[[:space:]]*${other}\$/port: ${want}/" "$MOONRAKER_CFG"; then + error_msg "Could not rewrite moonraker.conf - is /usr/data full?" + return 1 + fi fi for nginx_conf in "$NGINX_CONF_FILE" /etc/nginx/nginx.conf; do if [ -f "$nginx_conf" ]; then echo -e "Info: Pointing Nginx Moonraker upstream to port ${want}..." - sed -i "s/server 127\.0\.0\.1:${other};/server 127.0.0.1:${want};/" "$nginx_conf" + if ! sed -i "s/server 127\.0\.0\.1:${other};/server 127.0.0.1:${want};/" "$nginx_conf"; then + error_msg "Could not rewrite $nginx_conf!" + return 1 + fi fi done + return 0 } # Link the component in and enable it. Follows moonraker_timelapse.sh exactly: # the linked file is an untracked source file inside Moonraker's git repo, so # update_manager reports "Repo has untracked source files" forever unless it is # added to the repo's local exclude list. +# Returns non-zero rather than letting errexit escape. Under helper.sh's global +# `set -e` an unguarded `ln -sf` or `>>` failure - a full /usr/data is the +# classic K1 failure, users pack it with gcode - would exit the helper mid +# sequence, with both daemons stopped and no rollback and no menu. function nexusp_install_compat_component() { local repo_dir echo -e "Info: Linking Creality compatibility component..." - ln -sf "$CREALITY_COMPAT_URL" "$CREALITY_COMPAT_FILE" + if ! ln -sf "$CREALITY_COMPAT_URL" "$CREALITY_COMPAT_FILE" 2>/dev/null; then + error_msg "Could not link the compatibility component!" + return 1 + fi repo_dir="${CREALITY_COMPAT_FILE%/moonraker/components/creality_compat.py}" if [ -d "$repo_dir"/.git/info ]; then echo -e "Info: Excluding linked component from Moonraker repo..." @@ -295,8 +379,12 @@ function nexusp_install_compat_component() { # show for it - and that is the majority case, since install_moonraker_nginx # only rewrites moonraker.conf when Moonraker itself is reinstalled. echo -e "Info: Adding [creality_compat] to moonraker.conf file..." - printf '\n[creality_compat]\ngenerate_thumbnails: True\nlog_requests: False\n' >> "$MOONRAKER_CFG" + if ! printf '\n[creality_compat]\ngenerate_thumbnails: True\nlog_requests: False\n' >> "$MOONRAKER_CFG"; then + error_msg "Could not write to moonraker.conf - is /usr/data full?" + return 1 + fi fi + return 0 } function nexusp_remove_compat_component() { @@ -320,16 +408,26 @@ function nexusp_remove_compat_component() { # port back to 7126 unconditionally. On a retired box that left NOTHING # answering :7125 and the touchscreen dead with no error anywhere. function nexusp_reapply_retired_config() { - nexusp_set_moonraker_port 7125 7126 - nexusp_install_compat_component + # Chained explicitly: without the `&&` the function's status is whatever the + # component install returned, so a failed port rewrite would report success. + nexusp_set_moonraker_port 7125 7126 && nexusp_install_compat_component } # The merge, in either direction, with the dry run shown first. Returns non-zero # when it refuses; helper.sh sets -e globally and sources every script into the # same shell, so an unguarded call would abort the whole helper and drop the # user to a shell with no menu. Guarded at every call site. +# Never re-enable errexit before returning non-zero. `set -e` is a global shell +# option, not a function-local one, so `set -e; return 1` re-arms errexit and +# then hands the shell a failing simple command - the caller's `set +e` is +# already gone and the whole helper exits at the call site, at stage 3, with +# both daemons stopped, no message and no menu. That is the exact failure this +# function's guard exists to prevent, and it was verified in bash, sh, dash and +# zsh: the caller's error branch never ran. Restore errexit only on the paths +# that return 0, and let the call sites use `if ! nexusp_merge_history ...`, +# which is errexit-exempt by definition. function nexusp_merge_history() { - local direction="$1" python + local direction="$1" python rc merge_yn python="$(nexusp_python)" if [ ! -f "$MOONRAKER_DB" ] || [ ! -f "$NEXUSP_DB" ]; then echo -e "Info: Only one print history database exists, nothing to merge..." @@ -338,23 +436,45 @@ function nexusp_merge_history() { echo -e "Info: Print history merge (${direction}), dry run..." set +e "$python" "$MERGE_JOB_HISTORY_URL" --direction "$direction" - local rc=$? + rc=$? if [ "$rc" != "0" ]; then - set -e return "$rc" fi + # The dry run is only worth printing if somebody reads it. Running --apply + # straight afterwards made it decorative in the only context it is ever run + # from: the rows listed as NOT copied scroll past, and so does the backup + # path, which is the only rollback instruction that appears anywhere. + echo + echo -e " ${yellow}Read the list above. Rows marked 'in_progress (NOT copied)'" + echo -e " will not be carried across, and after this the other database is" + echo -e " no longer read. Note the backup path it prints.${white}" + echo + read -p " ${white}Apply this print history merge? (${yellow}y${white}/${yellow}n${white}): ${yellow}" merge_yn + echo -e "${white}" + case "${merge_yn}" in + Y|y) + ;; + *) + error_msg "Print history merge declined!" + return 1;; + esac echo -e "Info: Merging print history..." "$python" "$MERGE_JOB_HISTORY_URL" --direction "$direction" --apply rc=$? + if [ "$rc" != "0" ]; then + return "$rc" + fi set -e - return "$rc" + return 0 } # -------------------------------------------------------------------------- -# Rollback. Undoes stages 6-8 in reverse, and is only ever called from the -# failure paths below - a partial swap is the one outcome worth more code than -# the swap itself, because the symptom is a dead touchscreen and nothing in any -# log to connect it to this option. +# Rollback. Undoes stages 5-8 in reverse (port, rename, component), then +# restarts nexusp, Moonraker and nginx unconditionally - the restart is not +# stage-gated because every path that reaches here stopped both daemons at +# stages 2-3. Only ever called from the failure paths below: a partial swap is +# the one outcome worth more code than the swap itself, because the symptom is a +# dead touchscreen and nothing in any log to connect it to this option. # -------------------------------------------------------------------------- function nexusp_rollback_retire() { @@ -384,6 +504,42 @@ function nexusp_rollback_retire() { error_msg "Nexusp has NOT been retired - the printer is back as it was." } +function creality_compat_installed() { + [ -f "$CREALITY_COMPAT_FILE" ] +} + +# -------------------------------------------------------------------------- +# Repair paths +# -------------------------------------------------------------------------- + +# Re-apply the port and the component on a box that is renamed but unfinished, +# then verify. Shared by the interrupted-retirement path and by the +# firmware-resurrection repair, because in both cases the rename is already +# right and only the configuration needs putting back. +function nexusp_finish_retirement() { + echo -e "Info: Completing the retirement..." + if ! nexusp_reapply_retired_config; then + error_msg "Could not write the configuration - is /usr/data full?" + return 1 + fi + echo -e "Info: Restarting Moonraker service..." + stop_moonraker + start_moonraker + if ! nexusp_reload_nginx; then + echo -e "${yellow}Warning: Nginx could not be reloaded - Fluidd may 502 until${white}" + echo -e "${yellow}the printer reboots. The touchscreen is unaffected.${white}" + fi + if ! nexusp_verify_retired; then + error_msg "Moonraker is still not answering usably on port 7125!" + echo -e " ${darkred}Use Restore Nexusp Backend to put the touchscreen back.${white}" + return 1 + fi + ok_msg "The retirement has been completed!" + echo -e " ${cyan}Moonraker answers on 7125 and the touchscreen should${white}" + echo -e " ${cyan}reconnect on its own within a few seconds.${white}" + return 0 +} + # -------------------------------------------------------------------------- # Repair after a firmware update put the service file back # -------------------------------------------------------------------------- @@ -402,23 +558,27 @@ function nexusp_repair_resurrection() { fi nexusp_stop_service echo -e "Info: Re-applying the nexusp rename..." + # Both failure paths restart nexusp before returning. Without that, a repair + # that cannot write to the init directory leaves the daemon stopped as well as + # un-renamed - strictly worse than the state it was asked to fix, and it stays + # that way until the next reboot. if ! mv -f "$svc" "$disabled_svc" 2>/dev/null; then error_msg "Could not rename $(basename "$svc") - is $(dirname "$svc") writable?" + nexusp_start_service return fi if [ -f "$svc" ] || [ ! -f "$disabled_svc" ]; then error_msg "The nexusp service did not stay renamed!" + nexusp_start_service return fi - # The same update may also have reverted the port or the component. - nexusp_reapply_retired_config - echo -e "Info: Restarting Moonraker service..." - stop_moonraker - start_moonraker - nexusp_reload_nginx - ok_msg "The nexusp service has been disabled again!" - echo -e " ${cyan}Nothing else was changed; your history and settings are as they${white}" - echo -e " ${cyan}were before the firmware update.${white}" + # The same update may also have reverted the port or the component, so this + # goes through the shared finish path rather than assuming only the rename + # was lost. + if nexusp_finish_retirement; then + echo -e " ${cyan}Nothing else was changed; your history and settings are as${white}" + echo -e " ${cyan}they were before the firmware update.${white}" + fi } # -------------------------------------------------------------------------- @@ -433,7 +593,7 @@ function retire_nexusp(){ echo -e " binary and its database are never deleted, so Restore Nexusp" echo -e " Backend puts everything back.${white}" echo - local yn pillow_yn repair_yn svc disabled_svc answered + local yn pillow_yn repair_yn svc disabled_svc NEXUSP_RETIRE_STAGE=0 while true; do read -p "${white} Are you sure you want to retire ${green}Nexusp Backend ${white}? (${yellow}y${white}/${yellow}n${white}): ${yellow}" yn @@ -467,11 +627,32 @@ function retire_nexusp(){ return fi if ! nexusp_present; then - if nexusp_retired; then - error_msg "Nexusp Backend is already retired!" - else + if ! nexusp_retired; then error_msg "No nexusp service was found on this firmware!" + return + fi + # Retired, but is it FINISHED? A retirement interrupted between the + # rename and the port move - a dropped SSH session during the merge + # prompt is the realistic way - leaves nexusp disabled with Moonraker + # still on 7126 and nothing owning :7125. Refusing here would make + # this option decline to finish its own half-done work, and only + # Restore would recover, which nothing tells the user. So finish it. + if [ "$(nexusp_moonraker_port)" = "7125" ] && creality_compat_installed; then + error_msg "Nexusp Backend is already retired!" + return fi + echo -e " ${yellow}Nexusp is disabled, but the swap was not completed - Moonraker" + echo -e " is not on port 7125 and/or the compatibility component is" + echo -e " missing, so nothing is answering the touchscreen.${white}" + echo + read -p " ${white}Finish the retirement now? (${yellow}y${white}/${yellow}n${white}): ${yellow}" repair_yn + echo -e "${white}" + case "${repair_yn}" in + Y|y) + nexusp_finish_retirement;; + *) + error_msg "Left as it is - use Restore Nexusp Backend to go back.";; + esac return fi if ! creality_confirm_printer_idle; then @@ -500,9 +681,18 @@ function retire_nexusp(){ case "${pillow_yn}" in Y|y) + # Pinned, and binary-only first. This is Python 3.8 on MIPS, where + # there is no manylinux wheel, so an unpinned install resolves to + # whatever PyPI serves today and then runs that sdist's setup.py as + # root on the printer. Try a wheel first; fall back to the sdist + # only after saying so. echo -e "Info: Installing Pillow..." set +e - "$MOONRAKER_ENV_PYTHON" -m pip install Pillow + "$MOONRAKER_ENV_PYTHON" -m pip install --only-binary :all: 'Pillow==9.5.0' + if [ "$?" != "0" ]; then + echo -e "${yellow}No prebuilt wheel for this board - building from source.${white}" + "$MOONRAKER_ENV_PYTHON" -m pip install 'Pillow==9.5.0' + fi if [ "$?" != "0" ]; then echo -e "${yellow}Warning: Pillow could not be installed. Continuing without it -${white}" echo -e "${yellow}thumbnails already on disk are still listed.${white}" @@ -524,10 +714,10 @@ function retire_nexusp(){ NEXUSP_RETIRE_STAGE=3 # 4. The merge. Nothing has been renamed or re-pointed yet, so a refusal - # here just puts the daemons back. - set +e - nexusp_merge_history to-moonraker - if [ "$?" != "0" ]; then + # here just puts the daemons back. `if !` rather than `$?`: a command + # in an if-condition is errexit-exempt, so the guard cannot be + # defeated by the callee's own errexit state. + if ! nexusp_merge_history to-moonraker; then set -e error_msg "The print history merge refused to run - nothing was changed." echo -e " ${darkred}See the message above. Retiring nexusp without it would leave${white}" @@ -537,13 +727,15 @@ function retire_nexusp(){ start_moonraker return fi - set -e NEXUSP_RETIRE_STAGE=4 # 5. The component. Safe to install while nexusp is still enabled: it is # inert until Moonraker loads it. - nexusp_install_compat_component NEXUSP_RETIRE_STAGE=5 + if ! nexusp_install_compat_component; then + nexusp_rollback_retire + return + fi # 6. The rename, guarded. An unguarded mv would abort the whole helper # under set -e, leaving both daemons stopped with no message and no @@ -574,23 +766,33 @@ function retire_nexusp(){ fi NEXUSP_RETIRE_STAGE=6 - # 7-8. The port, on both sides at once. - nexusp_set_moonraker_port 7125 7126 + # 7-8. The port, on both sides at once. The stage moves BEFORE the call, + # so a failure partway through (moonraker.conf rewritten, nginx.conf + # not) is still rollback-visible. NEXUSP_RETIRE_STAGE=8 + if ! nexusp_set_moonraker_port 7125 7126; then + nexusp_rollback_retire + return + fi # 9. Start, and reload nginx explicitly rather than trusting the verb. echo -e "Info: Starting Moonraker service..." start_moonraker - nexusp_reload_nginx + if ! nexusp_reload_nginx; then + echo -e "${yellow}Warning: Nginx could not be reloaded, so it is still${white}" + echo -e "${yellow}proxying the old port. Moonraker is fine; Fluidd and${white}" + echo -e "${yellow}Mainsail will return 502 until Nginx restarts or the${white}" + echo -e "${yellow}printer reboots. The touchscreen talks to Moonraker${white}" + echo -e "${yellow}directly and is unaffected.${white}" + fi NEXUSP_RETIRE_STAGE=9 - # 10. Verify something actually answers on the port the screen polls. - set +e - "$CURL" -s -m 5 http://127.0.0.1:7125/server/info | grep -q '"result"' - answered=$? - set -e - if [ "$answered" != "0" ]; then - error_msg "Nothing answered on port 7125 after the swap!" + # 10. Verify the screen's port answers AND the component actually loaded. + # "Something answers 7125" is not the success condition - Moonraker + # answers 7125 perfectly well with the component in failed_components + # and the file browser dead. + echo -e "Info: Waiting for Moonraker to come up on port 7125..." + if ! nexusp_verify_retired; then nexusp_rollback_retire return fi @@ -649,9 +851,7 @@ function restore_nexusp(){ # un-merge the forward direction: those rows are valid Moonraker rows # either way and rolling them back would delete records that have no # other copy. - set +e - nexusp_merge_history to-nexusp - if [ "$?" != "0" ]; then + if ! nexusp_merge_history to-nexusp; then set -e error_msg "The print history merge refused to run - nothing was changed." echo -e " ${darkred}See the message above. Restoring without it would hand the${white}" @@ -659,11 +859,17 @@ function restore_nexusp(){ start_moonraker return fi - set -e - # Reverse of 8, then 7, then 6. - nexusp_set_moonraker_port 7126 7125 - nexusp_remove_compat_component + # The RENAME FIRST, verified, and only then the port and the component. + # + # The obvious order - port back to 7126, drop the component, then put the + # service back - has the same hole retire's step 6 guards against, in the + # other direction: if the rename fails after the port has moved, nothing + # owns :7125. Moonraker is on 7126, nexusp is still called `disabled.*` + # so it never starts, Fluidd works, and the touchscreen is dead with the + # error message talking about file permissions rather than about the port + # nobody is listening on. Doing the fallible step first means a failure + # leaves the box exactly as retired, which is a working state. disabled_svc="$(nexusp_disabled_service)" svc="$(dirname "$disabled_svc")/$(basename "$disabled_svc" | sed 's/^disabled\.//')" if [ -f "$svc" ]; then @@ -673,17 +879,35 @@ function restore_nexusp(){ echo -e "Info: Restoring nexusp service..." if ! mv "$disabled_svc" "$svc" 2>/dev/null; then error_msg "Could not restore $(basename "$svc") - is $(dirname "$svc") writable?" + echo -e " ${darkred}Nothing was changed; the printer is still in the retired${white}" + echo -e " ${darkred}state and the touchscreen keeps working.${white}" + start_moonraker + nexusp_reload_nginx + return + fi + if [ -f "$svc" ] && [ ! -f "$disabled_svc" ]; then + : + else + error_msg "The nexusp service did not stay restored!" + mv "$svc" "$disabled_svc" 2>/dev/null || true start_moonraker + nexusp_reload_nginx return fi fi + # Only now, with the service back on disk, give up the port. + nexusp_set_moonraker_port 7126 7125 + nexusp_remove_compat_component + echo -e "Info: Starting Moonraker service..." start_moonraker nexusp_start_service nexusp_reload_nginx ok_msg "Nexusp Backend has been restored successfully!" echo -e " ${cyan}The touchscreen is back on nexusp, and Moonraker on 7126.${white}" + echo -e " ${cyan}If the screen stays blank, nexusp did not start: re-run Retire${white}" + echo -e " ${cyan}Nexusp Backend to put Moonraker back on 7125.${white}" return;; N|n) error_msg "Restoration canceled!" diff --git a/tests/test_retire_nexusp.py b/tests/test_retire_nexusp.py new file mode 100644 index 0000000..77cf771 --- /dev/null +++ b/tests/test_retire_nexusp.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +"""Offline checks for scripts/retire_nexusp.sh — no printer, no daemons. + + python3 -m pytest -q tests/ + +WHY THIS FILE EXISTS +-------------------- +retire_nexusp.sh runs as root and, in one uninterruptible sequence, stops two +daemons, rewrites a print-history database, renames an init script and re-points +two config files. Until this file existed the only automated check on any of it +was `bash -n`, which proves the syntax parses and nothing else. + +Two defects found by reading rather than running motivated it, and both are +pinned below: + + 1. THE ERREXIT CONTRACT. `set -e; return 1` re-arms errexit and then hands the + caller a failing command — helper.sh sets -e globally and sources every + script into that same shell, so the whole helper exited at the call site, + at the point where both daemons are already stopped, with no message and no + menu to return to. The guard that was supposed to catch it never ran. + 2. ABSENCE MUST NOT BE AN ERROR. A predicate loop that finds nothing returns + the last failed `[ -f ]`. `svc="$(nexusp_enabled_service)"` then aborts the + helper under the same global errexit. + +Neither is visible in a diff and neither is catchable by any check this repo had. + +The sed round-trips are here for a different reason: the substitutions use `\\s`, +a GNU extension, and busybox sed on the printer may not honour it. Running them +against the REAL config files this repo ships is the only thing that would +surface that. + +⚠ Add cases as `test_*` functions. A file named test_*.py whose assertions live +in a hand-rolled runner gets collected and runs nothing while reporting green. +""" +import os +import shutil +import subprocess + +import pytest + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SCRIPT = os.path.join(REPO, "scripts", "retire_nexusp.sh") + + +def _gnu_sed_dir(tmp_path): + """A PATH entry whose `sed` accepts GNU `-i` with no argument, or None. + + The script uses `sed -i` the way every other script in this repo does, which + is GNU/busybox syntax. BSD sed (the macOS default) requires an argument + there, so on a Mac these tests would fail for a reason that has nothing to + do with the code. Use gsed when it is installed, and skip honestly when it + is not, rather than quietly not testing the substitutions. + """ + if subprocess.run(["sed", "--version"], capture_output=True).returncode == 0: + return None # already GNU, no shim needed + gsed = shutil.which("gsed") + if gsed is None: + pytest.skip("needs GNU sed (`brew install gnu-sed`) — the script uses " + "`sed -i` as busybox and GNU spell it") + # exist_ok / already-linked: several tests call run_sh more than once with + # the same tmp_path, and a second invocation must reuse the shim rather + # than blow up on it. + shim = tmp_path / "gnubin" + shim.mkdir(exist_ok=True) + link = shim / "sed" + if not link.exists(): + os.symlink(gsed, str(link)) + return str(shim) + + +def run_sh(body, tmp_path, want_sed=False): + """Source retire_nexusp.sh with every path pointed into tmp_path, run `body`. + + `set -e` is on, exactly as helper.sh has it, because that is the condition + half of these tests exist to check. + """ + env = dict(os.environ) + if want_sed: + shim = _gnu_sed_dir(tmp_path) + if shim: + env["PATH"] = shim + os.pathsep + env["PATH"] + initd = tmp_path / "initd" + initd.mkdir(exist_ok=True) + preamble = f""" +set -e +white=; yellow=; cyan=; green=; darkred=; red= +error_msg() {{ echo "ERR: $1"; }} +ok_msg() {{ echo "OK: $1"; }} +NEXUSP_SERVICE={initd}/CS56nexusp_service +NEXUSP_SERVICE_LEGACY={initd}/S56nexusp_service +MOONRAKER_CFG={tmp_path}/moonraker.conf +NGINX_CONF_FILE={tmp_path}/nginx.conf +MOONRAKER_DB={tmp_path}/moonraker-sql.db +NEXUSP_DB={tmp_path}/nexusp-sql.db +MOONRAKER_ENV_PYTHON={tmp_path}/nonexistent-python +MERGE_JOB_HISTORY_URL={tmp_path}/merge.py +CREALITY_COMPAT_FILE={tmp_path}/mr/moonraker/moonraker/components/creality_compat.py +CREALITY_COMPAT_URL={tmp_path}/source_component.py +. {SCRIPT} +""" + return subprocess.run(["bash", "-c", preamble + body], + capture_output=True, text=True, env=env, cwd=REPO) + + +# -------------------------------------------------------------------------- +# The errexit contract — the defect that killed the helper mid-retirement +# -------------------------------------------------------------------------- + +def test_a_refusing_merge_leaves_the_caller_in_control(tmp_path): + """The merge refuses (a daemon is alive, a schema drifted, the user says no) + and the CALLER must get to run its rollback. Before this was fixed the shell + exited inside the function, at stage 3, with Moonraker and nexusp both + stopped and the user dropped to a bare prompt.""" + (tmp_path / "moonraker-sql.db").write_text("") + (tmp_path / "nexusp-sql.db").write_text("") + (tmp_path / "merge.py").write_text("import sys\nsys.exit(1)\n") + r = run_sh(""" +if ! nexusp_merge_history to-moonraker; then + echo HANDLED + exit 0 +fi +echo UNEXPECTED_SUCCESS +exit 2 +""", tmp_path) + assert "HANDLED" in r.stdout, r.stdout + r.stderr + assert r.returncode == 0 + + +def test_declining_the_merge_prompt_is_reported_as_a_refusal(tmp_path): + """The confirmation gate: answering anything but y must abort the merge and + hand a non-zero status back, not fall through to --apply.""" + (tmp_path / "moonraker-sql.db").write_text("") + (tmp_path / "nexusp-sql.db").write_text("") + (tmp_path / "merge.py").write_text( + "import sys\n" + "open(%r, 'a').write(' '.join(sys.argv[1:]) + chr(10))\n" + % str(tmp_path / "calls.log")) + r = run_sh(""" +if ! echo n | nexusp_merge_history to-moonraker; then + echo DECLINED + exit 0 +fi +echo UNEXPECTED_APPLY +exit 2 +""", tmp_path) + assert "DECLINED" in r.stdout, r.stdout + r.stderr + calls = (tmp_path / "calls.log").read_text() + assert "--apply" not in calls, calls + + +def test_nothing_to_merge_is_not_a_refusal(tmp_path): + """One database missing means a printer that never ran both daemons. That + is a normal state, not an error, and must not abort the retirement.""" + (tmp_path / "moonraker-sql.db").write_text("") + r = run_sh(""" +if nexusp_merge_history to-moonraker; then echo CONTINUED; fi +""", tmp_path) + assert "CONTINUED" in r.stdout, r.stdout + r.stderr + + +# -------------------------------------------------------------------------- +# The predicates — absence is an answer, not an error +# -------------------------------------------------------------------------- + +STATE_PROBE = """ +state() { + p=no; r=no; a=no; x=no + nexusp_present && p=yes + nexusp_retired && r=yes + nexusp_absent && a=yes + nexusp_resurrected && x=yes + echo "present=$p retired=$r absent=$a resurrected=$x" +} +state +""" + + +def test_no_nexusp_at_all(tmp_path): + r = run_sh(STATE_PROBE, tmp_path) + assert "present=no retired=no absent=yes resurrected=no" in r.stdout + assert r.returncode == 0, "a predicate that finds nothing must not abort" + + +def test_nexusp_enabled(tmp_path): + (tmp_path / "initd").mkdir(exist_ok=True) + (tmp_path / "initd" / "CS56nexusp_service").write_text("#!/bin/sh\n") + r = run_sh(STATE_PROBE, tmp_path) + assert "present=yes retired=no absent=no resurrected=no" in r.stdout + + +def test_nexusp_retired(tmp_path): + (tmp_path / "initd").mkdir(exist_ok=True) + (tmp_path / "initd" / "disabled.CS56nexusp_service").write_text("#!/bin/sh\n") + r = run_sh(STATE_PROBE, tmp_path) + assert "present=no retired=yes absent=no resurrected=no" in r.stdout + + +def test_a_firmware_update_resurrecting_the_service_is_detected(tmp_path): + """Both forms present. /etc/init.d/rcK then starts the recreated one from + the CS pass while Moonraker starts from the S pass, so nexusp loses the + :7125 bind and dies silently at every boot.""" + (tmp_path / "initd").mkdir(exist_ok=True) + (tmp_path / "initd" / "CS56nexusp_service").write_text("#!/bin/sh\n") + (tmp_path / "initd" / "disabled.CS56nexusp_service").write_text("#!/bin/sh\n") + r = run_sh(STATE_PROBE, tmp_path) + assert "present=yes retired=yes absent=no resurrected=yes" in r.stdout + + +def test_the_legacy_s_prefix_is_recognised_too(tmp_path): + """The init script name varies by firmware; tools.sh already probes both.""" + (tmp_path / "initd").mkdir(exist_ok=True) + (tmp_path / "initd" / "S56nexusp_service").write_text("#!/bin/sh\n") + r = run_sh(STATE_PROBE, tmp_path) + assert "present=yes" in r.stdout + + +def test_an_absent_service_does_not_abort_a_variable_assignment(tmp_path): + """The specific shape that bit: command substitution in an assignment is a + simple command, so a non-zero function status kills the shell under -e.""" + r = run_sh(""" +svc="$(nexusp_enabled_service)" +disabled="$(nexusp_disabled_service)" +port="$(nexusp_moonraker_port)" +echo "SURVIVED svc=[$svc] disabled=[$disabled] port=[$port]" +""", tmp_path) + assert "SURVIVED svc=[] disabled=[] port=[]" in r.stdout, r.stdout + r.stderr + + +def test_a_config_without_a_port_line_reads_as_unknown(tmp_path): + """grep finding nothing must not abort the helper — the pipe into sed is + what keeps the status zero, so a refactor that drops it would be caught.""" + (tmp_path / "moonraker.conf").write_text("[server]\nhost: 0.0.0.0\n") + r = run_sh('port="$(nexusp_moonraker_port)"; echo "PORT=[$port]"', tmp_path) + assert "PORT=[]" in r.stdout, r.stdout + r.stderr + + +# -------------------------------------------------------------------------- +# The port swap, against the config files this repo actually ships +# -------------------------------------------------------------------------- + +def _real_configs(tmp_path): + shutil.copy(os.path.join(REPO, "files", "moonraker", "moonraker.conf"), + str(tmp_path / "moonraker.conf")) + (tmp_path / "nginx.conf").write_text( + "upstream apiserver {\n server 127.0.0.1:7126;\n}\n") + + +def test_the_port_swap_round_trips_byte_for_byte(tmp_path): + """7126 -> 7125 -> 7126 must land exactly where it started. The shipped + moonraker.conf says 7125, so the fixture starts it at 7126 the way a + configured printer has it.""" + _real_configs(tmp_path) + conf = tmp_path / "moonraker.conf" + conf.write_text(conf.read_text().replace("port: 7125", "port: 7126")) + before = conf.read_text(), (tmp_path / "nginx.conf").read_text() + r = run_sh("nexusp_set_moonraker_port 7125 7126 > /dev/null\n" + "nexusp_set_moonraker_port 7126 7125 > /dev/null\n" + "echo DONE", tmp_path, want_sed=True) + assert "DONE" in r.stdout, r.stdout + r.stderr + assert (conf.read_text(), (tmp_path / "nginx.conf").read_text()) == before + + +def test_the_port_swap_moves_moonraker_and_nginx_together(tmp_path): + """They must always agree. A box where they disagree serves 502s from + Fluidd and nothing says why.""" + _real_configs(tmp_path) + conf = tmp_path / "moonraker.conf" + conf.write_text(conf.read_text().replace("port: 7125", "port: 7126")) + r = run_sh("nexusp_set_moonraker_port 7125 7126 > /dev/null\n" + 'echo "PORT=$(nexusp_moonraker_port)"', tmp_path, want_sed=True) + assert "PORT=7125" in r.stdout, r.stdout + r.stderr + assert "server 127.0.0.1:7125;" in (tmp_path / "nginx.conf").read_text() + + +def test_the_port_swap_is_idempotent(tmp_path): + """Called twice in the same direction, the second call is a no-op — which + is what makes it safe for configure_moonraker_nginx_k1_2025 to re-run it on + every Moonraker reinstall.""" + _real_configs(tmp_path) + run_sh("nexusp_set_moonraker_port 7125 7126 > /dev/null", tmp_path, + want_sed=True) + once = (tmp_path / "moonraker.conf").read_text() + run_sh("nexusp_set_moonraker_port 7125 7126 > /dev/null", tmp_path, + want_sed=True) + assert (tmp_path / "moonraker.conf").read_text() == once + + +# -------------------------------------------------------------------------- +# The compat component, on both moonraker.conf branches +# -------------------------------------------------------------------------- + +def _component_tree(tmp_path): + (tmp_path / "mr" / "moonraker" / "moonraker" / "components").mkdir(parents=True) + (tmp_path / "mr" / "moonraker" / ".git" / "info").mkdir(parents=True) + (tmp_path / "mr" / "moonraker" / ".git" / "info" / "exclude").write_text("") + (tmp_path / "source_component.py").write_text("# component\n") + + +def test_the_shipped_conf_block_is_uncommented_and_recommented(tmp_path): + """The `[timelapse]` convention: the block ships commented and the install + uncomments it. A round trip must leave the file usable either way.""" + _component_tree(tmp_path) + shutil.copy(os.path.join(REPO, "files", "moonraker", "moonraker.conf"), + str(tmp_path / "moonraker.conf")) + conf = tmp_path / "moonraker.conf" + assert "#[creality_compat]" in conf.read_text(), "shipped conf must carry it" + r = run_sh("nexusp_install_compat_component > /dev/null\n" + 'grep -c "^\\[creality_compat\\]" ' + str(conf), tmp_path, + want_sed=True) + assert r.stdout.strip().endswith("1"), r.stdout + r.stderr + assert os.path.islink(str(tmp_path / "mr" / "moonraker" / "moonraker" / + "components" / "creality_compat.py")) + run_sh("nexusp_remove_compat_component > /dev/null", tmp_path, want_sed=True) + assert "#[creality_compat]" in conf.read_text() + assert not os.path.exists(str(tmp_path / "mr" / "moonraker" / "moonraker" / + "components" / "creality_compat.py")) + + +def test_a_conf_predating_this_option_gets_the_block_appended(tmp_path): + """The majority case. install_moonraker_nginx only rewrites moonraker.conf + when Moonraker itself is reinstalled, so most users' configs have no block + to uncomment — and without the append branch the component would be linked + and never loaded, leaving the file browser broken with nothing to show.""" + _component_tree(tmp_path) + (tmp_path / "moonraker.conf").write_text("[server]\nport: 7126\n\n[history]\n\n") + r = run_sh("nexusp_install_compat_component > /dev/null\n" + 'grep -c "^\\[creality_compat\\]" ' + + str(tmp_path / "moonraker.conf"), tmp_path, want_sed=True) + assert r.stdout.strip().endswith("1"), r.stdout + r.stderr + assert "generate_thumbnails: True" in (tmp_path / "moonraker.conf").read_text() + + +def test_installing_twice_adds_one_block_and_one_exclude_line(tmp_path): + """Idempotency matters because configure_moonraker_nginx_k1_2025 re-runs + this on every Moonraker reinstall of a retired box.""" + _component_tree(tmp_path) + (tmp_path / "moonraker.conf").write_text("[server]\nport: 7126\n\n") + run_sh("nexusp_install_compat_component > /dev/null", tmp_path, want_sed=True) + run_sh("nexusp_install_compat_component > /dev/null", tmp_path, want_sed=True) + conf = (tmp_path / "moonraker.conf").read_text() + exclude = (tmp_path / "mr" / "moonraker" / ".git" / "info" / "exclude").read_text() + assert conf.count("[creality_compat]") == 1 + assert exclude.count("creality_compat.py") == 1 + + +def test_removal_drops_the_git_exclude_entry(tmp_path): + """Symmetric with install: the entry exists so update_manager stops + reporting the repo as dirty, and it has no business outliving the link.""" + _component_tree(tmp_path) + (tmp_path / "moonraker.conf").write_text("[server]\nport: 7126\n\n") + run_sh("nexusp_install_compat_component > /dev/null", tmp_path, want_sed=True) + run_sh("nexusp_remove_compat_component > /dev/null", tmp_path, want_sed=True) + exclude = (tmp_path / "mr" / "moonraker" / ".git" / "info" / "exclude").read_text() + assert "creality_compat.py" not in exclude + + +# -------------------------------------------------------------------------- +# Write failures are reported, not left to errexit +# +# A full /usr/data is the classic K1 failure — users pack it with gcode. Under +# helper.sh's global `set -e` an unguarded `sed -i` or `>>` failure exits the +# whole helper mid-swap: both daemons stopped, no rollback, no menu. +# -------------------------------------------------------------------------- + +@pytest.mark.skipif(os.geteuid() == 0, reason="root ignores directory permissions") +def test_an_unwritable_conf_fails_the_port_swap_instead_of_the_helper(tmp_path): + """The DIRECTORY has to be read-only, not the file: `sed -i` writes a temp + beside the target and renames over it, so the target's own mode does not + stop it. A full or read-only /usr/data presents exactly this way.""" + locked = tmp_path / "locked" + locked.mkdir() + shutil.copy(os.path.join(REPO, "files", "moonraker", "moonraker.conf"), + str(locked / "moonraker.conf")) + locked.chmod(0o555) + try: + r = run_sh(f""" +MOONRAKER_CFG={locked}/moonraker.conf +NGINX_CONF_FILE={tmp_path}/no-such-nginx.conf +if ! nexusp_set_moonraker_port 7126 7125; then echo REPORTED; exit 0; fi +echo SILENT_SUCCESS; exit 2 +""", tmp_path, want_sed=True) + finally: + locked.chmod(0o755) + assert "REPORTED" in r.stdout, r.stdout + r.stderr + + +def test_an_unlinkable_component_fails_the_install_instead_of_the_helper(tmp_path): + """No components directory at all — the shape a partially extracted or + removed Moonraker leaves behind.""" + (tmp_path / "source_component.py").write_text("# component\n") + (tmp_path / "moonraker.conf").write_text("[server]\nport: 7126\n\n") + r = run_sh(""" +if ! nexusp_install_compat_component; then echo REPORTED; exit 0; fi +echo SILENT_SUCCESS; exit 2 +""", tmp_path) + assert "REPORTED" in r.stdout, r.stdout + r.stderr + + +def test_a_failed_nginx_reload_is_reported(tmp_path): + """nginx is the only thing the browser talks to. A reload that quietly fails + leaves it proxying the now-dead old port, so Fluidd 502s while Moonraker + itself is perfectly healthy — and probing Moonraker directly cannot see it.""" + r = run_sh(""" +NGINX_BIN=/nonexistent/nginx +if ! nexusp_reload_nginx; then echo REPORTED; exit 0; fi +echo SILENT_SUCCESS; exit 2 +""", tmp_path) + assert "REPORTED" in r.stdout, r.stdout + r.stderr + + +# -------------------------------------------------------------------------- +# The /server/info verification, which decides whether a swap is rolled back +# -------------------------------------------------------------------------- + +def test_a_component_in_failed_components_is_detected(tmp_path): + """Moonraker loads optional components with `load_component(config, section, + None)`, which swallows any exception into failed_components and KEEPS + SERVING. So /server/info answering 200 is not the success condition — an + earlier version treated it as one and reported a dead file browser as a + successful retirement.""" + body = ('{"result": {"components": ["file_manager", "history"], ' + '"failed_components": ["creality_compat"]}}') + r = run_sh(f""" +if nexusp_compat_load_failed '{body}'; then echo FAILED_DETECTED; else echo MISSED; fi +""", tmp_path) + assert "FAILED_DETECTED" in r.stdout, r.stdout + r.stderr + + +def test_a_healthy_component_is_not_read_as_failed(tmp_path): + """The component name appears in the healthy list too, so a naive substring + match on the whole body would report every good install as broken.""" + body = ('{"result": {"components": ["file_manager", "creality_compat"], ' + '"failed_components": []}}') + r = run_sh(f""" +if nexusp_compat_load_failed '{body}'; then echo FALSE_ALARM; else echo HEALTHY; fi +""", tmp_path) + assert "HEALTHY" in r.stdout, r.stdout + r.stderr + + +def test_another_components_failure_is_not_attributed_to_this_one(tmp_path): + body = ('{"result": {"components": ["creality_compat"], ' + '"failed_components": ["spoolman"]}}') + r = run_sh(f""" +if nexusp_compat_load_failed '{body}'; then echo MISATTRIBUTED; else echo HEALTHY; fi +""", tmp_path) + assert "HEALTHY" in r.stdout, r.stdout + r.stderr From 70225a8c9b3b48d291132f87e5bd0e9fbda79a5d Mon Sep 17 00:00:00 2001 From: arlophoenix Date: Mon, 3 Aug 2026 18:53:00 +1200 Subject: [PATCH 4/4] Close the failure paths Codex found A cross-model pass over the previous commit found two regressions it had introduced and several gaps neither the specialists nor the red team saw. Renumbering could not handle non-positive job ids. SQLite's INTEGER PRIMARY KEY is a signed rowid alias, so ids like [-2, -1] staged into [0, 1] and the second pass collided with itself: UNIQUE constraint failed, the transaction rolled back, and a valid database became unmergeable with the retirement aborting at the point both daemons are stopped. Staging now offsets by max(current_max, N), which is disjoint from the final range by construction. Making two helpers report write failures turned every unguarded caller into an abort under the global errexit - including two inside nexusp_rollback_retire itself, so a failed rollback could exit halfway and leave precisely the dead-port state it exists to prevent. Rollback is now unconditionally best-effort and reports what the printer actually looks like afterwards. Restore announced success without checking that nexusp started. The init script's status is discarded by design, so a failed start left nothing on port 7125 - Moonraker having already moved off it - under a success message. It now polls, and says how to recover. Prompts no longer kill the helper on EOF: read returns non-zero on a closed stdin and errexit did the rest, mid-retirement in the worst case. A signal handler now rolls back on SIGINT/SIGTERM/SIGHUP, so a dropped SSH session during the merge prompt no longer leaves nexusp disabled and the port unmoved. The merge also verifies its own backup (integrity check plus row count, then fsync) and refuses to apply a plan the databases have moved out from under. 147 tests pass. --- .../creality-compat/merge_job_history.py | 75 +++++++++++- .../creality-compat/test_merge_job_history.py | 76 ++++++++++++ scripts/retire_nexusp.sh | 111 +++++++++++++++--- tests/test_retire_nexusp.py | 45 +++++++ 4 files changed, 288 insertions(+), 19 deletions(-) diff --git a/files/moonraker/creality-compat/merge_job_history.py b/files/moonraker/creality-compat/merge_job_history.py index 45c6815..fea136e 100644 --- a/files/moonraker/creality-compat/merge_job_history.py +++ b/files/moonraker/creality-compat/merge_job_history.py @@ -368,8 +368,24 @@ def renumber_by_start_time(con): Done as an offset pass rather than in place: job_id is `INTEGER PRIMARY KEY ASC`, so assigning 1..N directly would collide with rows that still hold - those ids. Shifting every row above the current maximum first makes the - second pass collision-free without needing a temp table. + those ids. Shifting every row into a staging range first makes the second + pass collision-free without needing a temp table. + + The staging offset is `max(current_max, N)`, and BOTH terms are load-bearing. + `current_max` alone is wrong whenever the greatest existing id is negative - + SQLite allows that, since INTEGER PRIMARY KEY is a signed rowid alias. For + ids [-2, -1] the offset would be -1, staging becomes [0, 1], and the second + pass then tries to move 0 onto 1 while a row is already sitting there: + `UNIQUE constraint failed`. The transaction rolls back safely, but a + perfectly valid database becomes unmergeable and the retirement aborts with + a traceback at the point where both daemons are stopped. + + With offset = max(current_max, N): + - staging is offset+1 .. offset+N, every value > current_max, so it cannot + collide with an id not yet moved; + - the final range is 1..N, every value <= N <= offset < offset+1, so it + cannot collide with a staged id. + Disjoint by construction, for any input. Ordered by (start_time, job_id) so rows sharing a timestamp keep their existing relative order rather than being permuted arbitrarily. @@ -378,8 +394,9 @@ def renumber_by_start_time(con): "select job_id from job_history order by start_time, job_id").fetchall() if not rows: return - offset = con.execute( + current_max = con.execute( "select coalesce(max(job_id), 0) from job_history").fetchone()[0] + offset = max(current_max, len(rows)) for position, row in enumerate(rows, start=1): con.execute("update job_history set job_id = ? where job_id = ?", (offset + position, row[0])) @@ -483,8 +500,60 @@ def listing(rows): "stop it before merging; its cached job_totals would land on " "top of the merge at the next print." % alive) + # Re-read and re-classify now that the liveness guard has passed. + # + # Everything above was computed from a snapshot taken BEFORE that check, and + # the daemons are stopped asynchronously (start-stop-daemon -K then sleep 1, + # plus a bare killall). A daemon finishing its shutdown flush in that window + # closes the last print and writes its job_totals contribution - and the + # stale plan would then insert a row the target already has, or write + # pre-flush totals back over it. + # + # Deliberately ABORT on any difference rather than silently applying the + # newer plan: the operator just approved a specific set of counts, and + # quietly doing something else is the lie this script's own dry run exists + # to prevent. + fresh_target, _ = load_jobs(into_db) + fresh_source, _ = load_jobs(source_db) + fresh = classify(fresh_source, fresh_target) + if [len(bucket) for bucket in fresh] != [len(new), len(dupes), len(skipped)]: + sys.exit( + "refusing: the databases changed between the dry run and now\n" + "(was %d insert / %d duplicate / %d skipped, now %d / %d / %d).\n" + "Something is still writing to them. Stop it and run this again." + % (len(new), len(dupes), len(skipped), + len(fresh[0]), len(fresh[1]), len(fresh[2]))) + backup = "%s.bak-merge-%s" % (into_db, time.strftime("%Y%m%d_%H%M%S")) shutil.copy2(into_db, backup) + # The backup is the ENTIRE rollback story, and until it is checked it is + # only a file with a reassuring name. A copy interrupted by a full + # /usr/data leaves a truncated database indistinguishable from a good one. + # Verify it opens, passes an integrity check, and holds the rows we counted, + # then force it to disk - an fsync-less copy can be invalidated by a power + # cut even after the filesystem reported the write complete. + try: + check = sqlite3.connect("file:%s?mode=ro" % backup, uri=True) + try: + status = check.execute("pragma integrity_check").fetchone()[0] + backed_up = check.execute( + "select count(*) from job_history").fetchone()[0] + finally: + check.close() + except sqlite3.Error as why: + os.remove(backup) + sys.exit("refusing: the backup could not be re-opened (%s). Nothing " + "was written." % why) + if status != "ok" or backed_up != len(target_rows): + os.remove(backup) + sys.exit("refusing: the backup is not a faithful copy (integrity=%s, " + "%d of %d rows). Nothing was written." + % (status, backed_up, len(target_rows))) + fd = os.open(backup, os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) # Name the daemon that actually owns the file being replaced. The forward # merge writes Moonraker's database and the reverse one writes nexusp's, so # a fixed "stop Moonraker" tells the user to stop the wrong daemon on the diff --git a/files/moonraker/creality-compat/test_merge_job_history.py b/files/moonraker/creality-compat/test_merge_job_history.py index 81945e3..ec9bda3 100644 --- a/files/moonraker/creality-compat/test_merge_job_history.py +++ b/files/moonraker/creality-compat/test_merge_job_history.py @@ -402,6 +402,62 @@ def test_an_unreadable_cmdline_is_skipped_not_fatal(tmp_path): assert "nexusp" in REAL_MOONRAKER_RUNNING(proc) +def test_a_corrupt_backup_aborts_before_the_first_write(monkeypatch, capsys, + tmp_path): + """The backup is the ENTIRE rollback story, and until it is checked it is + just a file with a reassuring name. A copy cut short by a full /usr/data is + indistinguishable from a good one by filename alone — and restoring it + would destroy the history it was meant to protect.""" + into = make_db(tmp_path / "into.db", [job(T0)]) + source = make_db(tmp_path / "src.db", [job(T0 - 86400, filename="old.gcode")]) + + def truncating_copy(src, dst): + with open(dst, "wb") as fh: + fh.write(b"SQLite format 3\x00truncated") + + monkeypatch.setattr(mjh.shutil, "copy2", truncating_copy) + with pytest.raises(SystemExit) as exc: + run_main(monkeypatch, capsys, into, source, "--apply") + assert "backup" in str(exc.value) + assert len(rows_of(into)) == 1, "the target must be untouched" + assert glob.glob(into + ".bak-merge-*") == [], "the bad backup must be removed" + + +def test_a_database_changing_under_the_merge_aborts(monkeypatch, capsys, tmp_path): + """Everything is classified from a snapshot taken before the liveness check, + and the daemons stop asynchronously. If one finishes a shutdown flush in + that window the approved plan is stale — and applying it anyway would insert + a row the target now has, or write pre-flush totals back over it.""" + into = make_db(tmp_path / "into.db", [job(T0)]) + source = make_db(tmp_path / "src.db", [job(T0 - 86400, filename="old.gcode")]) + real_load = mjh.load_jobs + calls = [] + + def load_then_mutate(db): + result = real_load(db) + calls.append(db) + if len(calls) == 2: # after the dry-run pair, before the re-read + # A row that CHANGES THE PLAN: the daemon's shutdown flush lands the + # very print the merge was about to insert, so the approved + # "1 insert" becomes "1 duplicate" and applying the stale plan would + # write a second copy. + con = sqlite3.connect(into) + j = job(T0 - 86400, filename="old.gcode") + cols = [c for c in j if c != "job_id"] + con.execute("insert into job_history (%s) values (%s)" + % (",".join(cols), ",".join("?" * len(cols))), + tuple(j[c] for c in cols)) + con.commit() + con.close() + return result + + monkeypatch.setattr(mjh, "load_jobs", load_then_mutate) + with pytest.raises(SystemExit) as exc: + run_main(monkeypatch, capsys, into, source, "--apply") + assert "changed between the dry run" in str(exc.value) + assert glob.glob(into + ".bak-merge-*") == [] + + def test_a_missing_database_exits_before_touching_anything(monkeypatch, capsys, tmp_path): into = make_db(tmp_path / "into.db", [job(T0)]) @@ -694,6 +750,26 @@ def test_rows_sharing_a_start_time_keep_their_relative_order(monkeypatch, capsys assert by_id == ["old.gcode", "first.gcode", "second.gcode"] +@pytest.mark.parametrize("ids", [[-2, -1], [-100, -1], [-1, 0], [0, 1]]) +def test_renumbering_handles_non_positive_existing_ids(monkeypatch, capsys, + tmp_path, ids): + """SQLite's INTEGER PRIMARY KEY is a signed rowid alias, so negative and + zero ids are legal. Staging at `max(job_id) + n` alone puts the staging + range on top of the final 1..N range whenever the maximum is negative, and + the second pass then collides: `UNIQUE constraint failed`. The transaction + rolls back safely, but a valid database becomes unmergeable and the + retirement aborts with a traceback at the point both daemons are stopped. + """ + into = make_db(tmp_path / "into.db", + [job(T0 + i, job_id=jid, filename="f%d.gcode" % i) + for i, jid in enumerate(ids)]) + source = make_db(tmp_path / "src.db", [job(T0 - 86400, filename="old.gcode")]) + run_main(monkeypatch, capsys, into, source, "--apply") + rows = rows_of(into, order="job_id") + assert [r["job_id"] for r in rows] == list(range(1, len(ids) + 2)) + assert rows[0]["filename"] == "old.gcode" + + def test_renumbering_survives_a_second_run(monkeypatch, capsys, tmp_path): """Idempotency still holds: the second pass inserts nothing and the ids it assigns are the ones already there.""" diff --git a/scripts/retire_nexusp.sh b/scripts/retire_nexusp.sh index ac2db04..abb567a 100644 --- a/scripts/retire_nexusp.sh +++ b/scripts/retire_nexusp.sh @@ -236,6 +236,22 @@ function nexusp_compat_load_failed() { esac } +# Poll until ANYTHING answers :7125 usefully. Used by the restore path, where +# the daemon that has to come up is Creality's, not ours, so the component check +# below does not apply - but "did it actually start" very much does. +function nexusp_wait_for_port_7125() { + local attempt + attempt=0 + while [ "$attempt" -lt 30 ]; do + if "$CURL" -s -m 5 http://127.0.0.1:7125/server/info 2>/dev/null | grep -q '"result"'; then + return 0 + fi + attempt=$((attempt + 1)) + sleep 2 + done + return 1 +} + # Poll until Moonraker answers on :7125 AND has actually loaded the component. # # Polling, not a single probe: start_moonraker sleeps one second, and Moonraker's @@ -449,7 +465,7 @@ function nexusp_merge_history() { echo -e " will not be carried across, and after this the other database is" echo -e " no longer read. Note the backup path it prints.${white}" echo - read -p " ${white}Apply this print history merge? (${yellow}y${white}/${yellow}n${white}): ${yellow}" merge_yn + read -p " ${white}Apply this print history merge? (${yellow}y${white}/${yellow}n${white}): ${yellow}" merge_yn || merge_yn="n" echo -e "${white}" case "${merge_yn}" in Y|y) @@ -477,30 +493,50 @@ function nexusp_merge_history() { # dead touchscreen and nothing in any log to connect it to this option. # -------------------------------------------------------------------------- +# EVERY step here is best-effort. A rollback that can itself abort is worse than +# no rollback at all: it turns a detected failure into the exact persistent +# dead-port state it exists to prevent, halfway through undoing it. So each step +# is `|| true`, errexit is off for the duration, and the outcome is reported +# from what the printer actually looks like afterwards rather than assumed. function nexusp_rollback_retire() { - local disabled_svc svc + local disabled_svc svc failed + failed="" echo echo -e "${yellow}Rolling back...${white}" + set +e if [ "$NEXUSP_RETIRE_STAGE" -ge 8 ]; then - nexusp_set_moonraker_port 7126 7125 + nexusp_set_moonraker_port 7126 7125 || failed="1" fi if [ "$NEXUSP_RETIRE_STAGE" -ge 6 ]; then disabled_svc="$(nexusp_disabled_service)" if [ -n "$disabled_svc" ]; then svc="$(dirname "$disabled_svc")/$(basename "$disabled_svc" | sed 's/^disabled\.//')" - mv "$disabled_svc" "$svc" 2>/dev/null || true + mv "$disabled_svc" "$svc" 2>/dev/null || failed="1" fi fi if [ "$NEXUSP_RETIRE_STAGE" -ge 5 ]; then - nexusp_remove_compat_component + nexusp_remove_compat_component || failed="1" fi nexusp_start_service start_moonraker - nexusp_reload_nginx + nexusp_reload_nginx || true + set -e # The forward history merge is NOT undone, deliberately. Those rows are valid # Moonraker rows either way, and rolling them back would delete records that # exist in no other database - the exact loss this whole option is careful # about. Nothing else read them, so leaving them costs nothing. + if [ -n "$failed" ]; then + error_msg "The rollback did not fully complete!" + echo -e " ${darkred}Part of the change could not be undone - most likely /usr/data${white}" + echo -e " ${darkred}is full or read-only. Check that the touchscreen still works.${white}" + echo -e " ${darkred}Moonraker port is now: $(nexusp_moonraker_port), nexusp is${white}" + if nexusp_present; then + echo -e " ${darkred}enabled.${white}" + else + echo -e " ${darkred}still disabled - run Restore Nexusp Backend.${white}" + fi + return + fi error_msg "Nexusp has NOT been retired - the printer is back as it was." } @@ -596,7 +632,7 @@ function retire_nexusp(){ local yn pillow_yn repair_yn svc disabled_svc NEXUSP_RETIRE_STAGE=0 while true; do - read -p "${white} Are you sure you want to retire ${green}Nexusp Backend ${white}? (${yellow}y${white}/${yellow}n${white}): ${yellow}" yn + read -p "${white} Are you sure you want to retire ${green}Nexusp Backend ${white}? (${yellow}y${white}/${yellow}n${white}): ${yellow}" yn || yn="n" case "${yn}" in Y|y) echo -e "${white}" @@ -616,7 +652,7 @@ function retire_nexusp(){ echo -e " Moonraker at every boot. The repair is to re-apply the rename," echo -e " keeping the file the update wrote.${white}" echo - read -p " ${white}Disable the recreated ${green}nexusp service ${white}again? (${yellow}y${white}/${yellow}n${white}): ${yellow}" repair_yn + read -p " ${white}Disable the recreated ${green}nexusp service ${white}again? (${yellow}y${white}/${yellow}n${white}): ${yellow}" repair_yn || repair_yn="n" echo -e "${white}" case "${repair_yn}" in Y|y) @@ -645,7 +681,7 @@ function retire_nexusp(){ echo -e " is not on port 7125 and/or the compatibility component is" echo -e " missing, so nothing is answering the touchscreen.${white}" echo - read -p " ${white}Finish the retirement now? (${yellow}y${white}/${yellow}n${white}): ${yellow}" repair_yn + read -p " ${white}Finish the retirement now? (${yellow}y${white}/${yellow}n${white}): ${yellow}" repair_yn || repair_yn="n" echo -e "${white}" case "${repair_yn}" in Y|y) @@ -669,7 +705,7 @@ function retire_nexusp(){ echo -e " so installing it fixes both. It is a large download and there may" echo -e " be no prebuilt wheel for this board.${white}" echo - read -p " ${white}Install ${green}Pillow ${white}into Moonraker's virtualenv? (${yellow}y${white}/${yellow}n${white}): ${yellow}" pillow_yn + read -p " ${white}Install ${green}Pillow ${white}into Moonraker's virtualenv? (${yellow}y${white}/${yellow}n${white}): ${yellow}" pillow_yn || pillow_yn="n" echo -e "${white}" fi @@ -700,6 +736,15 @@ function retire_nexusp(){ set -e;; esac + # From here on the printer is being mutated, so arm a signal handler. + # helper.sh's only trap resets the terminal colour, and SIGHUP keeps its + # default terminate-immediately behaviour - so a dropped SSH or WiFi + # session anywhere below would kill the helper with both daemons stopped + # and, past step 6, with nexusp disabled and the port not yet moved: + # nothing on :7125 and no rollback. The window includes an unbounded + # "read the merge list" prompt, which is exactly where a user walks away. + # Cleared on every exit path below. + trap 'echo; echo -e "${yellow}Interrupted - rolling back...${white}"; nexusp_rollback_retire; trap - INT TERM HUP; exit 1' INT TERM HUP NEXUSP_RETIRE_STAGE=1 # 1. Anything that can restart Moonraker behind our back has to be off # BEFORE the merge, not just the daemon. Nothing in this repo can, so @@ -725,6 +770,7 @@ function retire_nexusp(){ echo -e " ${darkred}the helper script.${white}" nexusp_start_service start_moonraker + trap - INT TERM HUP return fi NEXUSP_RETIRE_STAGE=4 @@ -734,6 +780,7 @@ function retire_nexusp(){ NEXUSP_RETIRE_STAGE=5 if ! nexusp_install_compat_component; then nexusp_rollback_retire + trap - INT TERM HUP return fi @@ -748,12 +795,14 @@ function retire_nexusp(){ echo -e " ${darkred}avoid losing the backup. Delete whichever copy you do not want${white}" echo -e " ${darkred}and run this option again.${white}" nexusp_rollback_retire + trap - INT TERM HUP return fi echo -e "Info: Disabling nexusp service..." if ! mv "$svc" "$disabled_svc" 2>/dev/null; then error_msg "Could not rename $(basename "$svc") - is $(dirname "$svc") writable?" nexusp_rollback_retire + trap - INT TERM HUP return fi # VERIFIED, not assumed. If the rename silently did not take and the @@ -762,6 +811,7 @@ function retire_nexusp(){ if [ -f "$svc" ] || [ ! -f "$disabled_svc" ]; then error_msg "The nexusp service did not stay renamed!" nexusp_rollback_retire + trap - INT TERM HUP return fi NEXUSP_RETIRE_STAGE=6 @@ -772,6 +822,7 @@ function retire_nexusp(){ NEXUSP_RETIRE_STAGE=8 if ! nexusp_set_moonraker_port 7125 7126; then nexusp_rollback_retire + trap - INT TERM HUP return fi @@ -794,6 +845,7 @@ function retire_nexusp(){ echo -e "Info: Waiting for Moonraker to come up on port 7125..." if ! nexusp_verify_retired; then nexusp_rollback_retire + trap - INT TERM HUP return fi @@ -805,6 +857,7 @@ function retire_nexusp(){ echo -e " ${cyan}resolves itself and needs no action.${white}" echo -e " ${cyan}A firmware update can put the nexusp service back - the menus${white}" echo -e " ${cyan}report it, and re-running this option repairs it.${white}" + trap - INT TERM HUP return;; N|n) error_msg "Retiring canceled!" @@ -896,18 +949,44 @@ function restore_nexusp(){ fi fi - # Only now, with the service back on disk, give up the port. - nexusp_set_moonraker_port 7126 7125 - nexusp_remove_compat_component + # Only now, with the service back on disk, give up the port. Guarded: + # these report write failures rather than letting errexit escape, and an + # unguarded call here would exit the helper with nexusp renamed-but-not- + # started and Moonraker still stopped - nothing on :7125 at all. + if ! nexusp_set_moonraker_port 7126 7125; then + error_msg "Could not move Moonraker back to port 7126!" + echo -e " ${darkred}Undoing: the printer stays retired and keeps working.${white}" + set +e + nexusp_set_moonraker_port 7125 7126 + mv "$svc" "$disabled_svc" 2>/dev/null + set -e + start_moonraker + nexusp_reload_nginx || true + return + fi + nexusp_remove_compat_component || true echo -e "Info: Starting Moonraker service..." start_moonraker nexusp_start_service - nexusp_reload_nginx + nexusp_reload_nginx || true + + # Verify, do not assume. nexusp_start_service discards the init script's + # status by design, so without this the option announces success while + # the touchscreen's only backend failed to start and NOTHING is on + # :7125 - Moonraker has already moved off it. + echo -e "Info: Waiting for nexusp to come up on port 7125..." + if ! nexusp_wait_for_port_7125; then + error_msg "nexusp did not answer on port 7125 after restarting!" + echo -e " ${darkred}The touchscreen has no backend. Its database was restored and${white}" + echo -e " ${darkred}is intact.${white}" + echo -e " ${cyan}Run Retire Nexusp Backend to put Moonraker back on 7125,${white}" + echo -e " ${cyan}or reboot the printer to let nexusp start from init.${white}" + echo + return + fi ok_msg "Nexusp Backend has been restored successfully!" echo -e " ${cyan}The touchscreen is back on nexusp, and Moonraker on 7126.${white}" - echo -e " ${cyan}If the screen stays blank, nexusp did not start: re-run Retire${white}" - echo -e " ${cyan}Nexusp Backend to put Moonraker back on 7125.${white}" return;; N|n) error_msg "Restoration canceled!" diff --git a/tests/test_retire_nexusp.py b/tests/test_retire_nexusp.py index 77cf771..adfb276 100644 --- a/tests/test_retire_nexusp.py +++ b/tests/test_retire_nexusp.py @@ -408,6 +408,51 @@ def test_a_failed_nginx_reload_is_reported(tmp_path): assert "REPORTED" in r.stdout, r.stdout + r.stderr +def test_a_closed_stdin_cancels_rather_than_killing_the_helper(tmp_path): + """`read` returns non-zero on EOF, and under helper.sh's global set -e that + exits the whole helper with no message. Piping input, running over a dropped + SSH session, or any non-interactive invocation hits it. Failing to read an + answer is a "no", not a crash.""" + (tmp_path / "initd").mkdir(exist_ok=True) + (tmp_path / "initd" / "CS56nexusp_service").write_text("#!/bin/sh\n") + (tmp_path / "mr").mkdir(exist_ok=True) + r = run_sh(f""" +top_line(){{ :; }}; inner_line(){{ :; }}; hr(){{ :; }}; bottom_line(){{ :; }}; title(){{ :; }} +MOONRAKER_FOLDER={tmp_path}/mr +retire_nexusp < /dev/null +echo SURVIVED +""", tmp_path) + assert "SURVIVED" in r.stdout, r.stdout + r.stderr + assert "canceled" in r.stdout + + +def test_the_rollback_cannot_abort_partway_through(tmp_path): + """A rollback that can itself exit under errexit turns a detected failure + into the persistent dead-port state it exists to prevent, halfway through + undoing it. Every step is best-effort and the outcome is reported from what + the printer looks like afterwards.""" + (tmp_path / "initd").mkdir(exist_ok=True) + locked = tmp_path / "locked" + locked.mkdir() + shutil.copy(os.path.join(REPO, "files", "moonraker", "moonraker.conf"), + str(locked / "moonraker.conf")) + locked.chmod(0o555) + try: + r = run_sh(f""" +MOONRAKER_CFG={locked}/moonraker.conf +NGINX_CONF_FILE={tmp_path}/no-such-nginx.conf +start_moonraker(){{ echo "(moonraker restarted)"; }} +NEXUSP_RETIRE_STAGE=8 +nexusp_rollback_retire +echo REACHED_THE_END +""", tmp_path, want_sed=True) + finally: + locked.chmod(0o755) + assert "REACHED_THE_END" in r.stdout, r.stdout + r.stderr + assert "(moonraker restarted)" in r.stdout, "daemons must be restarted anyway" + assert "did not fully complete" in r.stdout, "and the shortfall must be said" + + # -------------------------------------------------------------------------- # The /server/info verification, which decides whether a swap is rolled back # --------------------------------------------------------------------------