diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 3560a93af3..7024351758 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -98,6 +98,8 @@ Below are the dev container defaults. The field name to change these defaults is - Plugin loader: enabled; reads `.devcontainer/config/plugin-config.py` if present - If `plugin-config.py` is missing: plugin is enabled with empty config (features won’t work until configured) + + ## πŸ”§ Configuration ### NetBox Version and Environment (use .devcontainer/.env) diff --git a/.devcontainer/scripts/diagnose.sh b/.devcontainer/scripts/diagnose.sh index 133e6ca97f..be7596d699 100755 --- a/.devcontainer/scripts/diagnose.sh +++ b/.devcontainer/scripts/diagnose.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script echo "πŸ” DevContainer Startup Diagnostics" echo "==================================" diff --git a/.devcontainer/scripts/load-aliases.sh b/.devcontainer/scripts/load-aliases.sh index 65149d6198..feac6ee98f 100755 --- a/.devcontainer/scripts/load-aliases.sh +++ b/.devcontainer/scripts/load-aliases.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script # Quick alias loader for current session # Usage: source .devcontainer/scripts/load-aliases.sh diff --git a/.devcontainer/scripts/setup.sh b/.devcontainer/scripts/setup.sh index 588fe652ab..7f4278fd46 100755 --- a/.devcontainer/scripts/setup.sh +++ b/.devcontainer/scripts/setup.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script set -e echo "πŸš€ Setting up NetBox LibreNMS Plugin development environment..." diff --git a/.devcontainer/scripts/start-netbox.sh b/.devcontainer/scripts/start-netbox.sh index 789dcb845a..d5e4796600 100755 --- a/.devcontainer/scripts/start-netbox.sh +++ b/.devcontainer/scripts/start-netbox.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script # Check if we should run in background or foreground BACKGROUND=false @@ -18,7 +19,6 @@ if [ "$CODESPACES" = "true" ] && [ -n "$CODESPACE_NAME" ]; then echo "πŸ”— GitHub Codespaces detected" else ACCESS_URL="http://localhost:8000" - echo "πŸ› Debug: ACCESS_URL is set to: $ACCESS_URL" fi # Load shared process management helpers diff --git a/.devcontainer/scripts/welcome.sh b/.devcontainer/scripts/welcome.sh index 9328d663aa..e273313766 100755 --- a/.devcontainer/scripts/welcome.sh +++ b/.devcontainer/scripts/welcome.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script # Ensure aliases are available in the postAttach terminal session source "$(dirname "$0")/load-aliases.sh" 2>/dev/null @@ -44,7 +45,7 @@ if [ -n "$CODESPACES" ]; then echo " πŸ’‘ Click the link in the Ports panel or look for the 'Open in Browser' button" else echo "πŸ–₯️ Local Development Environment:" - echo " NetBox will be available at: http://localhost:8000 (paste into you browser)" + echo " NetBox will be available at: http://localhost:8000 (paste into your browser)" fi echo "" diff --git a/.github/workflows/lint-format.yaml b/.github/workflows/lint-format.yaml index 3e12242f63..055f809cc5 100644 --- a/.github/workflows/lint-format.yaml +++ b/.github/workflows/lint-format.yaml @@ -2,13 +2,7 @@ name: Lint and Format on: push: - branches: - - master - - develop pull_request: - branches: - - master - - develop jobs: format-and-lint: @@ -20,8 +14,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: '3.9' - cache: 'pip' + python-version: '3.12' - name: Install dependencies run: | @@ -29,22 +22,7 @@ jobs: pip install ruff - name: Run Ruff linting - run: | - echo "::group::Ruff Linting" - ruff check . --output-format=github - echo "::endgroup::" + run: ruff check . - name: Run Ruff formatting check - run: | - echo "::group::Ruff Formatting" - ruff format --check . - echo "::endgroup::" - - - name: Report formatting issues - if: failure() - run: | - echo "::error::Formatting or linting issues detected!" - echo "To fix locally, run:" - echo " ruff check --fix ." - echo " ruff format ." - echo "Then commit and push the changes." + run: ruff format --check . diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000000..8714ac5342 --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,28 @@ +# Contrib: Example Mapping Files + +This directory contains example YAML mapping files for bulk import into the +NetBox LibreNMS Plugin. Each file can be imported via the plugin's bulk import +feature in the NetBox UI. + +## How to Import + +1. Navigate to the mapping page (e.g., **LibreNMS β†’ Device Type Mappings**) +2. Click the **Import** button (upload icon) in the top right +3. Select **YAML** format +4. Paste the contents of the relevant YAML file +5. Click **Submit** + +## Available Mappings + +| File | Description | +|------|-------------| +| `interface_type_mappings.yaml` | Maps LibreNMS interface types + speeds to NetBox interface types | +| `device_type_mappings.yaml` | Maps LibreNMS hardware strings to NetBox device types | +| `module_type_mappings.yaml` | Maps LibreNMS inventory model names to NetBox module types (incl. transceivers) | +| `module_bay_mappings.yaml` | Maps LibreNMS inventory container names to NetBox module bay names | + +## Customisation + +These files are **examples** β€” adjust values to match the device types, module +types, and interface types defined in your NetBox instance. The `netbox_*` +fields must reference objects that already exist in your NetBox. diff --git a/contrib/device_type_mappings.yaml b/contrib/device_type_mappings.yaml new file mode 100644 index 0000000000..2dec241524 --- /dev/null +++ b/contrib/device_type_mappings.yaml @@ -0,0 +1,73 @@ +# Device Type Mappings +# +# Maps LibreNMS hardware strings to NetBox device types. +# Import via: LibreNMS Plugin > Device Type Mappings > Import +# +# Fields: +# librenms_hardware β€” Hardware string exactly as shown in LibreNMS +# netbox_device_type β€” NetBox DeviceType (matched by model name or ID) +# description β€” Optional note +# +# The librenms_hardware value is matched case-insensitively. +# These mappings are checked BEFORE the built-in part_number/model fallback. + +# Juniper β€” LibreNMS reports verbose marketing names +- librenms_hardware: "Juniper MX480 Internet Backbone Router" + netbox_device_type: "MX480" + description: "Juniper MX480 chassis" + +- librenms_hardware: "Juniper MX960 Internet Backbone Router" + netbox_device_type: "MX960" + description: "Juniper MX960 chassis" + +- librenms_hardware: "Juniper MX304 Edge Router" + netbox_device_type: "MX304" + description: "Juniper MX304 edge router" + +- librenms_hardware: "JNP10008 [PTX10008]" + netbox_device_type: "PTX10008" + description: "Juniper PTX10008 core router" + +- librenms_hardware: "JNP7100-32C [ACX7100-32C]" + netbox_device_type: "ACX7100-32C" + description: "Juniper ACX7100-32C" + +- librenms_hardware: "JNP7024 [ACX7024]" + netbox_device_type: "ACX7024" + description: "Juniper ACX7024" + +- librenms_hardware: "Juniper JNP10008 Internet Backbone Router" + netbox_device_type: "PTX10008" + description: "Juniper PTX10008 (alternate hardware string)" + +- librenms_hardware: "Juniper VRR Internet Backbone Router" + netbox_device_type: "VRR" + description: "Juniper Virtual Route Reflector" + +# Nokia β€” model string matches directly in most cases +- librenms_hardware: "7750 SR-7s" + netbox_device_type: "7750 SR-7s" + description: "Nokia 7750 SR-7s service router" + +# Cisco β€” often matches by part_number but not always +- librenms_hardware: "WS-C4900M" + netbox_device_type: "WS-C4900M" + description: "Cisco Catalyst 4900M" + +# Cisco IOS XR +- librenms_hardware: "8201-SYS" + netbox_device_type: "8201" + description: "Cisco 8201 (hardware string differs from model)" + +# UfiSpace β€” LibreNMS reports SONiC/ONIE platform names +- librenms_hardware: "x86-64-ufispace-s9610-36d-r0" + netbox_device_type: "S9610-36D" + description: "UfiSpace S9610-36D" + +- librenms_hardware: "x86-64-ufispace-s9610-46dx-r0" + netbox_device_type: "S9610-46DX" + description: "UfiSpace S9610-46DX" + +- librenms_hardware: "x86-64-ufispace-s9700-53dx-r9" + netbox_device_type: "S9700-53DX" + description: "UfiSpace S9700-53DX" diff --git a/contrib/interface_name_rules.yaml b/contrib/interface_name_rules.yaml new file mode 100644 index 0000000000..52da69dff5 --- /dev/null +++ b/contrib/interface_name_rules.yaml @@ -0,0 +1,200 @@ +# Interface Name Rules +# +# Post-install interface rename rules for module types where NetBox's +# position-based naming can't produce the correct interface name. +# +# Covers two scenarios: +# 1. Converter offset β€” e.g., GLC-T inside CVR-X2-SFP needs port numbering +# that accounts for the converter's position in the parent module bay. +# 2. Breakout channels β€” e.g., QSFP+ 4x10G produces multiple sub-interfaces +# from a single physical port. +# +# Template variables: +# {slot} β€” Top-level slot/module bay position +# {bay_position} β€” Position of the bay this module is installed into (raw) +# {bay_position_num} β€” Numeric suffix of bay position (e.g., "swp1" β†’ "1") +# {parent_bay_position} β€” Position of the parent module's bay +# {sfp_slot} β€” Numeric sub-bay index within the parent module +# {base} β€” Original interface name from the NetBox module template +# {channel} β€” Breakout channel number (iterated) +# +# Arithmetic expressions are supported inside braces: +# {8 + ({parent_bay_position} - 1) * 2 + {sfp_slot}} +# +# Bulk import via: LibreNMS Plugin > Settings > Interface Name Rules > Import + +# --- Converter Offset Examples --- + +# SFP-1G-T (1G copper SFP, covers GLC-T/GLC-TE) in CVR-X2-SFP converter +# X2 bays are numbered 1-N; each converter holds 2 SFP slots +# Resulting interface: GigabitEthernet/ +- module_type: SFP-1G-T + parent_module_type: CVR-X2-SFP + name_template: "GigabitEthernet{slot}/{8 + ({parent_bay_position} - 1) * 2 + {sfp_slot}}" + channel_count: 0 + channel_start: 0 + description: "SFP-1G-T in CVR-X2-SFP: offset port numbering for X2-to-SFP conversion" + +# --- Breakout Channel Examples --- + +# QSFP-4X10G-LR breakout β€” Juniper-style (channels start at 0) +- module_type: QSFP-4X10G-LR + name_template: "{base}:{channel}" + channel_count: 4 + channel_start: 0 + description: "QSFP+ 4x10G-LR breakout with Juniper-style channel numbering (0-3)" + +# QSFP-4X10G-SR breakout β€” Juniper-style (channels start at 0) +- module_type: QSFP-4X10G-SR + name_template: "{base}:{channel}" + channel_count: 4 + channel_start: 0 + description: "QSFP+ 4x10G-SR breakout with Juniper-style channel numbering (0-3)" + +# --- Commented Examples --- + +# QSFP+ 4x10G breakout β€” Cisco-style (channels start at 1) +# - module_type: QSFP-4X10G-LR +# name_template: "{base}:{channel}" +# channel_count: 4 +# channel_start: 1 +# description: "QSFP+ 4x10G breakout with Cisco-style channel numbering (1-4)" + +# --- UfiSpace/Arcos Breakout Rules --- +# UfiSpace switches use swpNsC naming for breakout interfaces. +# bay_position_num extracts the numeric suffix from the bay name (e.g., "swp1" β†’ "1"). +# Channels start at 1, with 2 channels per 100G QSFP28 (2x100G breakout). + +# S9610-36D breakout rules +- module_type: QSFP-100G-LR4 + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SR4 + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SWDM4 + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-ZR + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" + +# S9610-46DX breakout rules +- module_type: QSFP-100G-LR4 + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SR4 + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SWDM4 + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-ZR + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" + +# S9700-53DX breakout rules +- module_type: QSFP-100G-LR4 + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SR4 + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SWDM4 + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-ZR + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" + +# --- Juniper ACX7024 Platform-Specific Rules --- +# These rules are scoped to the ACX7024 device type and use bay_position +# to generate Juniper-style interface names with FPC/PIC/port notation. + +# 100GE QSFP28 transceivers -> et-0/0/{port} +- module_type: QSFP-100G-LR4 + device_type: ACX7024 + name_template: "et-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 100GE QSFP28 naming" + +- module_type: QSFP-100G-SR4 + device_type: ACX7024 + name_template: "et-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 100GE QSFP28 naming" + +- module_type: QSFP-100G-SWDM4 + device_type: ACX7024 + name_template: "et-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 100GE QSFP28 naming" + +# 10GE SFP+ transceivers -> xe-0/0/{port} +- module_type: SFP-10G-SR + device_type: ACX7024 + name_template: "xe-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 10GE SFP+ naming" + +- module_type: SFP-10G-LR + device_type: ACX7024 + name_template: "xe-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 10GE SFP+ naming" + +# 1GE SFP transceivers -> ge-0/0/{port} +- module_type: SFP-1G-T + device_type: ACX7024 + name_template: "ge-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 1GE SFP naming" + +- module_type: SFP-1G-LX + device_type: ACX7024 + name_template: "ge-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 1GE SFP naming" diff --git a/contrib/interface_type_mappings.yaml b/contrib/interface_type_mappings.yaml new file mode 100644 index 0000000000..19db2a1fcf --- /dev/null +++ b/contrib/interface_type_mappings.yaml @@ -0,0 +1,70 @@ +# Interface Type Mappings +# +# Maps LibreNMS interface types (and optional speeds) to NetBox interface types. +# Import via: LibreNMS Plugin > Interface Mappings > Import +# +# Fields: +# librenms_type β€” IANA ifType string from LibreNMS (e.g. ethernetCsmacd) +# librenms_speed β€” Speed in Kbps (optional, null matches any speed) +# netbox_type β€” NetBox InterfaceTypeChoices slug +# description β€” Optional note +# +# Common NetBox interface type slugs: +# 1000base-t, 10gbase-t, 10gbase-x-sfpp, 25gbase-x-sfp28, +# 40gbase-x-qsfpp, 100gbase-x-qsfp28, 400gbase-x-qsfpdd, +# ieee802.11ax, lag, virtual, other + +- librenms_type: ethernetCsmacd + librenms_speed: 1000000 + netbox_type: 1000base-t + description: "1G Ethernet copper" + +- librenms_type: ethernetCsmacd + librenms_speed: 10000000 + netbox_type: 10gbase-x-sfpp + description: "10G Ethernet SFP+" + +- librenms_type: ethernetCsmacd + librenms_speed: 25000000 + netbox_type: 25gbase-x-sfp28 + description: "25G Ethernet SFP28" + +- librenms_type: ethernetCsmacd + librenms_speed: 40000000 + netbox_type: 40gbase-x-qsfpp + description: "40G Ethernet QSFP+" + +- librenms_type: ethernetCsmacd + librenms_speed: 100000000 + netbox_type: 100gbase-x-qsfp28 + description: "100G Ethernet QSFP28" + +- librenms_type: ethernetCsmacd + librenms_speed: 400000000 + netbox_type: 400gbase-x-qsfpdd + description: "400G Ethernet QSFP-DD" + +- librenms_type: ieee8023adLag + librenms_speed: + netbox_type: lag + description: "LACP/LAG aggregation" + +- librenms_type: propVirtual + librenms_speed: + netbox_type: virtual + description: "Virtual/loopback interface" + +- librenms_type: softwareLoopback + librenms_speed: + netbox_type: virtual + description: "Software loopback" + +- librenms_type: tunnel + librenms_speed: + netbox_type: virtual + description: "Tunnel interface" + +- librenms_type: l2vlan + librenms_speed: + netbox_type: virtual + description: "VLAN interface" diff --git a/contrib/module_bay_mappings.yaml b/contrib/module_bay_mappings.yaml new file mode 100644 index 0000000000..64c063176a --- /dev/null +++ b/contrib/module_bay_mappings.yaml @@ -0,0 +1,216 @@ +# Module Bay Mappings - Map LibreNMS inventory container names to NetBox module bay names +# +# These mappings replace heuristic matching between LibreNMS inventory and NetBox module bays. +# Import via: LibreNMS Plugin β†’ Module Bay Mappings β†’ Import +# +# Fields: +# librenms_name: LibreNMS entPhysicalName or container name (exact match or regex) +# librenms_class: Optional entPhysicalClass filter (powerSupply, fan, module, etc.) +# Leave empty for class-independent mappings +# netbox_bay_name: Target NetBox module bay name (supports \1, \2 backreferences with regex) +# is_regex: Set to true to treat librenms_name as a Python regex pattern +# description: Optional description +# +# Regex patterns use Python re.fullmatch() β€” the pattern must match the entire string. +# Backreferences (\1, \2) in netbox_bay_name reference capture groups in the pattern. + +# ─── Regex Patterns ────────────────────────────────────────────────────────── +# These patterns replace many individual exact-match entries. + +# Arcos/UfiSpace: sfpN β†’ Transceiver N (covers sfp0 through sfp53+) +- librenms_name: "^sfp(\\d+)$" + netbox_bay_name: "Transceiver \\1" + is_regex: true + description: "Arcos sfpN β†’ Transceiver N" + +# Cisco X2: Port Container slot/port β†’ X2 Port port +- librenms_name: "^Port Container (\\d+)/(\\d+)$" + netbox_bay_name: "X2 Port \\2" + is_regex: true + description: "Cisco X2 Port Container β†’ X2 Port N" + +# Cisco modules: Linecard/Supervisor(slot N) β†’ Slot N +- librenms_name: "^Linecard\\(slot (\\d+)\\)$" + librenms_class: "module" + netbox_bay_name: "Slot \\1" + is_regex: true + description: "Cisco Linecard slot β†’ Slot N" +- librenms_name: "^Supervisor\\(slot (\\d+)\\)$" + librenms_class: "module" + netbox_bay_name: "Slot \\1" + is_regex: true + description: "Cisco Supervisor slot β†’ Slot N" + +# Generic power supplies and fans +- librenms_name: "^Power Supply (\\d+)$" + librenms_class: "powerSupply" + netbox_bay_name: "PS\\1" + is_regex: true + description: "Power Supply N β†’ PSN" +- librenms_name: "^FanTray (\\d+)$" + librenms_class: "fan" + netbox_bay_name: "Fan Tray \\1" + is_regex: true + description: "FanTray N β†’ Fan Tray N" + +# Nokia 7750 SR chassis fans and power modules +- librenms_name: "^Chassis 1 Fan (\\d+)$" + librenms_class: "fan" + netbox_bay_name: "Fan \\1" + is_regex: true + description: "Nokia chassis fan β†’ Fan N" +- librenms_name: "^Chassis 1 PowShelf 1 PM (\\d+)$" + librenms_class: "powerSupply" + netbox_bay_name: "PM \\1" + is_regex: true + description: "Nokia power module β†’ PM N" + +# Nokia MDA and XIOM sub-module bays +# Bay names resolve from {module}/N templates: IOM Slot 1 pos=1 β†’ bay {module}/1 = 1/1 +- librenms_name: "^MDA (\\d+)/(\\d+)$" + librenms_class: "mdaModule" + netbox_bay_name: "\\1/\\2" + is_regex: true + description: "Nokia MDA N/M β†’ N/M (matches {module}/M on IOM)" +- librenms_name: "^XIOM (\\d+)/x(\\d+)$" + librenms_class: "xioModule" + netbox_bay_name: "\\1/x\\2" + is_regex: true + description: "Nokia XIOM N/xM β†’ N/xM (matches {module}/xM on IOM)" +- librenms_name: "^MDA (\\d+)/x(\\d+)/(\\d+)$" + librenms_class: "mdaModule" + netbox_bay_name: "x\\2/\\3" + is_regex: true + description: "Nokia MDA in XIOM N/xP/Q β†’ xP/Q (matches {module}/Q on XIOM)" + +# Nokia transceiver connector bays +# LibreNMS ifName "1/1/c1" (slot/mda/connector) β†’ NetBox bay "1/c1" +# ({module} on MDA resolves to position, stripping the slot prefix) +- librenms_name: "(\\d+)/(\\d+)/(c\\d+)" + librenms_class: "port" + netbox_bay_name: "\\2/\\3" + is_regex: true + description: "Nokia transceiver slot/mda/cN β†’ mda-pos/cN" +# LibreNMS ifName "2/x1/1/c2" (slot/xiom/mda/connector) β†’ NetBox bay "1/c2" +- librenms_name: "(\\d+)/x(\\d+)/(\\d+)/(c\\d+)" + librenms_class: "port" + netbox_bay_name: "\\3/\\4" + is_regex: true + description: "Nokia XIOM transceiver slot/xiom/mda/cN β†’ mda-pos/cN" + +# Juniper MX transceiver bays +# LibreNMS entPhysicalDescr format: "SFP+-10G-SR @ {fpc}/{pic}/{port}" +# NetBox MPC-3D-16XGE-SFPP bay format: "Transceiver {pic}/{port}" +- librenms_name: "[^@]+ @ \\d+/(\\d+)/(\\d+)" + librenms_class: "port" + netbox_bay_name: "Transceiver \\1/\\2" + is_regex: true + description: "Juniper MX SFP+ @ fpc/pic/port β†’ Transceiver pic/port" + +# ─── Exact Match Entries ───────────────────────────────────────────────────── +# These are for special cases where names don't follow a regex pattern. + +# Nokia CPM slots +- librenms_name: "Slot A" + librenms_class: "cpmModule" + netbox_bay_name: "Slot A" + description: "Nokia CPM slot A" +- librenms_name: "Slot B" + librenms_class: "cpmModule" + netbox_bay_name: "Slot B" + description: "Nokia CPM slot B" +- librenms_name: "SR-7s 2 CPM mini" + librenms_class: "cpmCarrier" + netbox_bay_name: "CMA" + description: "Nokia CMA2-7s CPM carrier bracket" + +# Juniper fixed-form devices +- librenms_name: "PSM 0" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 0" + description: "Juniper PSU slot 0" +- librenms_name: "PSM 1" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 1" + description: "Juniper PSU slot 1" + +# Juniper chassis devices (PTX10008 etc.): PSM β†’ PEM +# Regex runs after exact matches, so PSM 0/1 β†’ PSU 0/1 above takes priority for ACX +- librenms_name: "^PSM (\\d+)$" + librenms_class: "powerSupply" + netbox_bay_name: "PEM \\1" + is_regex: true + description: "Juniper chassis PSM N β†’ PEM N" + +# Juniper FPC container: "FPC: @ N/*/*" β†’ FPC N +- librenms_name: "^FPC: .+ @ (\\d+)/\\*/\\*$" + librenms_class: "container" + netbox_bay_name: "FPC \\1" + is_regex: true + description: "Juniper FPC container description β†’ FPC N" + +# Juniper transceivers: " @ slot/pic/port" description β†’ Transceiver slot/pic/port +- librenms_name: "^.+ @ (\\d+/\\d+/\\d+)$" + librenms_class: "port" + netbox_bay_name: "Transceiver \\1" + is_regex: true + description: "Juniper transceiver description β†’ Transceiver slot/pic/port" + +# Juniper fan trays: "Fan Tray N" β†’ "Fan N" (ACX7100, etc.) +# Runs after exact match, so "Fan Tray 0" β†’ "Fan Tray" (ACX7024) still works +- librenms_name: "^Fan Tray (\\d+)$" + librenms_class: "fan" + netbox_bay_name: "Fan \\1" + is_regex: true + description: "Juniper Fan Tray N β†’ Fan N (ACX7100 etc.)" + +# Juniper MX304: PEM β†’ PSU (MX304 bays are named PSU, not PEM) +- librenms_name: "PEM 0" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 0" + description: "Juniper MX304 PEM 0 β†’ PSU 0" +- librenms_name: "PEM 1" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 1" + description: "Juniper MX304 PEM 1 β†’ PSU 1" + +- librenms_name: "Fan Tray 0" + librenms_class: "fan" + netbox_bay_name: "Fan Tray" + description: "Juniper single fan tray (ACX7024)" + +# Juniper PTX10008: SIB β†’ CB (Switch Interface Board β†’ Component Board slot) +- librenms_name: "SIB 0" + librenms_class: "container" + netbox_bay_name: "CB 0" + description: "Juniper PTX10008 SIB 0 β†’ CB 0" +- librenms_name: "SIB 1" + librenms_class: "container" + netbox_bay_name: "CB 1" + description: "Juniper PTX10008 SIB 1 β†’ CB 1" +- librenms_name: "SIB 2" + librenms_class: "container" + netbox_bay_name: "CB 2" + description: "Juniper PTX10008 SIB 2 β†’ CB 2" +- librenms_name: "SIB 3" + librenms_class: "container" + netbox_bay_name: "CB 3" + description: "Juniper PTX10008 SIB 3 β†’ CB 3" +- librenms_name: "SIB 4" + librenms_class: "container" + netbox_bay_name: "CB 4" + description: "Juniper PTX10008 SIB 4 β†’ CB 4" +- librenms_name: "SIB 5" + librenms_class: "container" + netbox_bay_name: "CB 5" + description: "Juniper PTX10008 SIB 5 β†’ CB 5" + +# Arcos power supplies +- librenms_name: "psu0" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 0" + description: "Arcos PSU slot 0" +- librenms_name: "psu1" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 1" + description: "Arcos PSU slot 1" diff --git a/contrib/module_type_mappings.yaml b/contrib/module_type_mappings.yaml new file mode 100644 index 0000000000..e70d726f1b --- /dev/null +++ b/contrib/module_type_mappings.yaml @@ -0,0 +1,332 @@ +# Module Type Mappings +# +# Maps LibreNMS inventory model names (entPhysicalModelName) to NetBox module types. +# Import via: LibreNMS Plugin > Module Type Mappings > Import +# +# Fields: +# librenms_model β€” Model name from LibreNMS SNMP inventory +# netbox_module_type β€” NetBox ModuleType (matched by model name or ID) +# description β€” Optional note +# +# These mappings are checked FIRST. If no mapping exists, the plugin falls back +# to exact model name and part_number matching against NetBox module types. + +# ─── Cisco Catalyst 4900M ──────────────────────────────────────────────────── + +- librenms_model: "WS-X4908-10GE" + netbox_module_type: "WS-X4908-10GE" + description: "Cisco 8-port 10G X2 line card" + +- librenms_model: "WS-X4992" + netbox_module_type: "WS-X4992" + description: "Cisco 48-port 10/100/1000 line card" + +- librenms_model: "PWR-C49M-1000AC" + netbox_module_type: "PWR-C49M-1000AC" + description: "Cisco 1000W AC power supply" + +- librenms_model: "CVR-X2-SFP" + netbox_module_type: "CVR-X2-SFP" + description: "Cisco X2-to-SFP converter" + +# ─── Juniper Backplane ─────────────────────────────────────────────────────── + +- librenms_model: "710-017414" + netbox_module_type: "MX480-CHASSIS-BP" + description: "Juniper MX480 backplane (matched by part number)" + +- librenms_model: "CHAS-BP-MX480-S" + netbox_module_type: "MX480-CHASSIS-BP" + description: "Juniper MX480 backplane (matched by name)" + +# ─── Juniper FPC / Line Card Mappings ──────────────────────────────────────── +# Juniper FPCs use 750-xxxxxx part numbers as entPhysicalModelName. + +- librenms_model: "750-018124" + netbox_module_type: "DPCE-R-4XGE-XFP" + description: "Juniper DPCE 4-port 10G XFP DPC" + +- librenms_model: "750-022765" + netbox_module_type: "DPCE-R-20GE-2XGE" + description: "Juniper DPCE 20x1G + 2x10G combo DPC" + +- librenms_model: "750-028467" + netbox_module_type: "MPC-3D-16XGE-SFPP" + description: "Juniper MPC 16-port 10G SFP+" + +- librenms_model: "750-056519" + netbox_module_type: "MPC7E-MRATE" + description: "Juniper MPC7E 12-port QSFP+/QSFP28 multirate" + +- librenms_model: "750-062581" + netbox_module_type: "MPC-3D-16XGE-SFPP" + description: "Juniper MPC 16-port 10G SFP+ (variant PN)" + +# ─── Juniper Power Supply Mappings ─────────────────────────────────────────── + +- librenms_model: "740-029970" + netbox_module_type: "PWR-MX480-2520-AC" + description: "Juniper MX480 2520W AC PSU" + +- librenms_model: "740-063046" + netbox_module_type: "PWR-MX480-2520-AC" + description: "Juniper MX480 2520W AC PSU (variant PN)" + +- librenms_model: "740-027760" + netbox_module_type: "PWR-MX960-4100-AC" + description: "Juniper MX960 4100W AC PSU" + +- librenms_model: "740-110419" + netbox_module_type: "JNP-PWR2200-AC" + description: "Juniper MX304 2200W AC PSU" + +# Removed: JPSU-1600W-1UACAFO β€” exact model match, no mapping needed + +# ─── Juniper Fan Tray Mappings ─────────────────────────────────────────────── + +- librenms_model: "740-031521" + netbox_module_type: "FFANTRAY-MX960-HC" + description: "Juniper MX960 high-capacity fan tray" + +- librenms_model: "760-126744" + netbox_module_type: "JNP-FAN-2RU" + description: "Juniper MX304 2RU fan tray" + +# Removed: JNP7100-FAN1RU-AO β€” exact model match, no mapping needed + +# ─── Nokia 7750 SR-7s Module Mappings ──────────────────────────────────────── +# Nokia 3HE part numbers are handled by NormalizationRule: +# 1. Strip extra text (e.g. "3HE10550AARA01 NOK IPU3BFUEAA" β†’ "3HE10550AARA01") +# 2. Strip revision suffix (e.g. "3HE10550AARA01" β†’ "3HE10550AA") +# The normalized value matches the part_number field on NetBox ModuleTypes. +# No explicit Nokia mappings are needed. + +# ─── Transceiver Mappings: Juniper Part Numbers ───────────────────────────── +# Juniper-qualified optics use 740-xxxxxx part numbers regardless of OEM vendor. + +- librenms_model: "740-013111" + netbox_module_type: "SFP-1G-T" + description: "Juniper SFP 1000BASE-T copper" + +- librenms_model: "740-021308" + netbox_module_type: "SFP-10G-SR" + description: "Juniper SFP+ 10G-SR" + +- librenms_model: "740-031850" + netbox_module_type: "SFP-1G-LX" + description: "Juniper SFP 1000BASE-LX 10km" + +- librenms_model: "740-031980" + netbox_module_type: "SFP-10G-SR" + description: "Juniper SFP+ 10G-SR" + +- librenms_model: "740-031981" + netbox_module_type: "SFP-10G-LR" + description: "Juniper SFP+ 10G-LR" + +- librenms_model: "740-047682" + netbox_module_type: "CFP-100G-LR4" + description: "Juniper CFP 100G-LR4" + +- librenms_model: "740-054050" + netbox_module_type: "QSFP-4X10G-LR" + description: "Juniper QSFP+ 4x10G-LR" + +- librenms_model: "740-054053" + netbox_module_type: "QSFP-4X10G-SR" + description: "Juniper QSFP+ 4x10G-SR" + +- librenms_model: "740-058732" + netbox_module_type: "QSFP-100G-LR4" + description: "Juniper QSFP28 100G-LR4" + +- librenms_model: "740-061405" + netbox_module_type: "QSFP-100G-SR4" + description: "Juniper QSFP28 100G-SR4" + +- librenms_model: "740-061409" + netbox_module_type: "QSFP-100G-LR4" + description: "Juniper QSFP28 100G-LR4" + +- librenms_model: "740-079871" + netbox_module_type: "QSFP28-DD-2X100G-LR4" + description: "Juniper QSFP-DD 2x100G-LR4" + +- librenms_model: "740-082823" + netbox_module_type: "QSFP-DD-400G-LR8" + description: "Juniper QSFP-DD 400G-LR8" + +- librenms_model: "740-085349" + netbox_module_type: "QSFP-DD-400G-FR4" + description: "Juniper QSFP-DD 400G-FR4" + +- librenms_model: "740-085351" + netbox_module_type: "QSFP-DD-400G-DR4" + description: "Juniper QSFP-DD 400G-DR4" + +- librenms_model: "740-096176" + netbox_module_type: "QSFP-DD-400G-LR4" + description: "Juniper QSFP-DD 400G-LR4 (10km variant)" + +- librenms_model: "740-131169" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Juniper QSFP-DD 400G-ZR-M" + +- librenms_model: "740-151745" + netbox_module_type: "QSFP-DD-400G-ZR-M-HP" + description: "Juniper QSFP-DD 400G-ZR-M high-power" + +- librenms_model: "740-172665" + netbox_module_type: "QSFP-100G-ZR" + description: "Juniper QSFP28 100G-ZR" + +# ─── Transceiver Mappings: Finisar / II-VI / Coherent ──────────────────────── +# These are BASE part numbers (after normalization strips customer suffixes). +# See contrib/normalization_rules.yaml for the Finisar suffix-stripping rule. + +- librenms_model: "FTLC1154RDPL" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4" + +- librenms_model: "FTLC1151RDPL" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4 (variant)" + +- librenms_model: "FTLX1474D3BCL" + netbox_module_type: "SFP-10G-LR" + description: "Finisar SFP+ 10G-LR" + +- librenms_model: "FTCD3323R1PCL" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Finisar/II-VI QSFP-DD 400G-ZR-M coherent" + +- librenms_model: "FTLC9152RGPL" + netbox_module_type: "QSFP-100G-SWDM4" + description: "Finisar QSFP28 100G-SWDM4" + +# ─── Transceiver Mappings: Cisco / Cisco-branded OEM ───────────────────────── + +- librenms_model: "X2-10GB-LR" + netbox_module_type: "X2-10GB-LR" + description: "Cisco X2 10G-LR" + +- librenms_model: "X2-10GB-SR" + netbox_module_type: "X2-10GB-SR" + description: "Cisco X2 10G-SR" + +- librenms_model: "GLC-T" + netbox_module_type: "SFP-1G-T" + description: "Cisco SFP 1000BASE-T copper" + +- librenms_model: "GLC-TE" + netbox_module_type: "SFP-1G-T" + description: "Cisco SFP 1000BASE-T copper (extended temp)" + +- librenms_model: "SPP5200LR-C5" + netbox_module_type: "SFP-10G-LR" + description: "Cisco-branded Sumitomo SFP+ 10G-LR" + +- librenms_model: "SPP5310LR-C5" + netbox_module_type: "SFP-10G-LR" + description: "Cisco-branded Sumitomo SFP+ 10G-LR" + +- librenms_model: "SFBR-709SMZ-CS1" + netbox_module_type: "SFP-10G-SR" + description: "Cisco-branded Avago/Broadcom SFP+ 10G-SR" + +- librenms_model: "DP04QSDD-HE0" + netbox_module_type: "QSFP-DD-400G-ZR+" + description: "Cisco/Acacia QSFP-DD 400G-ZR+ coherent" + +- librenms_model: "QDD-400G-ZRP-S" + netbox_module_type: "QSFP-DD-400G-ZR+" + description: "Cisco QSFP-DD 400G-ZR+" + +- librenms_model: "QDD-400G-ZR4-S" + netbox_module_type: "QSFP-DD-400G-ZR" + description: "Cisco QSFP-DD 400G-ZR" + +# ─── Transceiver Mappings: Ciena ───────────────────────────────────────────── + +- librenms_model: "180-3530-900" + netbox_module_type: "QSFP-DD-400G-ZR" + description: "Ciena WaveLogic 5 Nano QSFP-DD 400ZR" + +- librenms_model: "176-3360-900" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Ciena QSFP-DD 400G-ZR-M coherent" + +- librenms_model: "176-3530-901" + netbox_module_type: "QSFP-DD-400G-ZR" + description: "Ciena QSFP-DD 400G-ZR coherent" + +- librenms_model: "176-3590-900" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Ciena QSFP-DD 400G-ZR-M coherent" + +# ─── Transceiver Mappings: T1 Nexus ───────────────────────────────────────── + +- librenms_model: "T1-QDD-400G-LR4" + netbox_module_type: "QSFP-DD-400G-LR4" + description: "T1 Nexus QSFP-DD 400G-LR4" + +- librenms_model: "T1-QDD-400G-FR4" + netbox_module_type: "QSFP-DD-400G-FR4" + description: "T1 Nexus QSFP-DD 400G-FR4" + +- librenms_model: "T1-QSFP28-LR4" + netbox_module_type: "QSFP-100G-LR4" + description: "T1 Nexus QSFP28 100G-LR4" + +- librenms_model: "100G-LR4_A3" + netbox_module_type: "QSFP-100G-LR4" + description: "T1 Nexus QSFP28 100G-LR4 (rev A3)" + +# ─── Transceiver Mappings: Innolight ──────────────────────────────────────── + +- librenms_model: "T-DQ4CNT-NCN" + netbox_module_type: "QSFP-DD-400G-FR4" + description: "Innolight QSFP-DD 400G-FR4" + +# ─── Transceiver Mappings: FS.com ──────────────────────────────────────────── + +- librenms_model: "Q28-PC03" + netbox_module_type: "QSFP28-100G-CU3M" + description: "FS.com QSFP28 100G passive DAC 3m" + +# ─── Transceiver Mappings: ProLabs ─────────────────────────────────────────── + +- librenms_model: "Q28LR431-10-IN" + netbox_module_type: "QSFP-100G-LR4" + description: "ProLabs QSFP28 100G-LR4 10km" + +# ─── Transceiver Mappings: Arcos Fixed-Port Part Numbers ───────────────────── + +- librenms_model: "SP7041-TE" + netbox_module_type: "SFP-1G-T" + description: "SFP 1000BASE-T copper (Arcos platform)" + +# ─── Transceiver Mappings: LeGrand Innolight ───────────────────────────────── + +- librenms_model: "LGI-FTLC9152RGPL" + netbox_module_type: "QSFP-100G-SWDM4" + description: "LeGrand-branded Finisar QSFP28 100G-SWDM4" + +# ─── Transceiver Mappings: Additional Finisar Variants ────────────────────── +# Some transceivers have customer-code suffixes that normalization may not handle. +# Add direct mappings as fallback. + +- librenms_model: "FTLC1151RDPL-CN" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4 (CN customer code)" + +- librenms_model: "FTLC1154RDPL-A5" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4 (A5 customer code)" + +# ─── Unknown / Unidentified Part Numbers ───────────────────────────────────── +# These are mapped based on port context (QSFP28 100G slot) when vendor is unknown. + +- librenms_model: "1F3QAA" + netbox_module_type: "QSFP-100G-LR4" + description: "Unknown QSFP28 100G (mapped by port context)" diff --git a/contrib/normalization_rules.yaml b/contrib/normalization_rules.yaml new file mode 100644 index 0000000000..c3d2081bea --- /dev/null +++ b/contrib/normalization_rules.yaml @@ -0,0 +1,61 @@ +# Normalization Rules β€” Examples +# +# Regex-based string transformations applied before module type, device type, +# or module bay matching. Rules run in priority order (lower first); each +# rule's output feeds the next. +# +# Import via: LibreNMS β†’ Normalization Rules β†’ Import β†’ YAML +# +# Fields: +# scope β€” module_type, device_type, or module_bay +# manufacturer β€” Optional manufacturer name (must exist in NetBox). +# When set, the rule only fires for that manufacturer. +# match_pattern β€” Python regex (re.sub pattern) +# replacement β€” Replacement string (supports \1, \2 back-references) +# priority β€” Lower values run first (default 100) +# description β€” Optional note + +# ── Nokia revision suffix stripping ────────────────────────────────────────── +# Nokia ENTITY-MIB reports module/transceiver models with 4-char revision +# suffixes (e.g. 3HE16474AARA01). NetBox module types use the base part +# number (3HE16474AA). This rule strips the suffix before matching. +# +# Captures the 10-char base (3HE + 5 alnum + 2 quality-tier letters), +# discards the 2-letter revision code + 2-digit build number. +- scope: module_type + manufacturer: Nokia + match_pattern: "^(3HE\\w{5}[A-Z]{2})[A-Z]{2}\\d{2}$" + replacement: "\\1" + priority: 100 + description: "Strip Nokia revision suffixes (e.g. RA01, RB01, RG01) from ENTITY-MIB model strings" + +# ── Finisar / II-VI / Coherent suffix stripping ───────────────────────────── +# Finisar part numbers have customer-specific suffixes after a hyphen: +# FTLC1154RDPL-A5 (original Finisar) +# FTLC1154RDPL-C (Prolabs compatible) +# FTLX1474D3BCL-C1 (Cisco-coded Finisar) +# This rule strips everything after the last hyphen for FT... models. +- scope: module_type + match_pattern: "^(FT[A-Z0-9]+)-[A-Z0-9]+$" + replacement: "\\1" + priority: 100 + description: "Strip Finisar/II-VI customer suffixes (-A5, -C, -CN, -C1, etc.)" + +# ── Prolabs LGI- prefix stripping ─────────────────────────────────────────── +# Prolabs-compatible optics sometimes prepend LGI- to the OEM part number: +# LGI-FTLC9152RGPL β†’ FTLC9152RGPL +- scope: module_type + match_pattern: "^LGI-(.+)$" + replacement: "\\1" + priority: 50 + description: "Strip Prolabs LGI- prefix from OEM part numbers" + +# ── Nokia transceiver model field cleanup ──────────────────────────────────── +# Nokia transceiver API sometimes returns model strings with trailing vendor +# info: "3HE10550AARA01 NOK IPU3BFUEAA" β€” extract just the part number. +- scope: module_type + manufacturer: Nokia + match_pattern: "^(3HE\\w+)\\s+.*$" + replacement: "\\1" + priority: 50 + description: "Extract Nokia part number from transceiver model field (strip trailing vendor/oui info)" diff --git a/docs/usage_tips/custom_field.md b/docs/usage_tips/custom_field.md index 032812ed82..7ed27a2f97 100644 --- a/docs/usage_tips/custom_field.md +++ b/docs/usage_tips/custom_field.md @@ -4,6 +4,9 @@ To enhance device identification and synchronization between NetBox and LibreNMS, this plugin supports using a custom field `librenms_id` on Device, Virtual Machine and Interface objects. While the plugin works without it, using this custom field is recommended for LibreNMS API lookups, and to assist with matching the remote device and remote interfaces for cable creation in Netbox. It can also be entered manually if no primary IP or FQDN is available. +!!! info "Automatic Creation" + As of version 0.4.2, the plugin **automatically creates** the `librenms_id` custom field when migrations are run. You no longer need to create it manually. The field is created for Device, Virtual Machine, Interface, and VM Interface objects. + For the Device and Virtual Machine objects the plugin will automatically populate the LibreNMS ID custom field when opening the LibreNMS Sync page if the device has been found in LibreNMS. For the Interface object, the plugin will automatically populate the LibreNMS ID custom field when the interface data is synced from LibreNMS. @@ -15,7 +18,10 @@ For the Interface object, the plugin will automatically populate the LibreNMS ID - **Efficient Synchronization:** Enhances the reliability of API lookups. - **Cable creation:** Allows better device identification for the creation of cables between NetBox devices. -## Suggested Custom Field Setup +## Manual Custom Field Setup (Legacy) + +!!! note + This section is only needed if you are running an older version of the plugin that does not auto-create the field, or if you need to recreate it after deletion. Follow these steps to create the `librenms_id` custom field in NetBox: diff --git a/docs/usage_tips/permissions.md b/docs/usage_tips/permissions.md index 9f9ecfb3f3..39c5225d2a 100644 --- a/docs/usage_tips/permissions.md +++ b/docs/usage_tips/permissions.md @@ -26,7 +26,7 @@ A user needs both tiers of permissions to complete an action. For example, to vi The Plugin also enforces Netbox object permissions so the following permission would also be required: -2. **Tier 2: Object permission**: User needs `dcim.add_device` (to create the device in NetBox) +1. **Tier 2: Object permission**: User needs `dcim.add_device` (to create the device in NetBox) If either permission is missing, the operation fails with an appropriate error message. diff --git a/netbox_librenms_plugin/__init__.py b/netbox_librenms_plugin/__init__.py index f1720c85d6..206a3009c8 100644 --- a/netbox_librenms_plugin/__init__.py +++ b/netbox_librenms_plugin/__init__.py @@ -28,6 +28,7 @@ def ready(self): super().ready() from django.conf import settings + from django.db.models.signals import post_migrate plugin_config = getattr(settings, "PLUGINS_CONFIG", {}).get(self.name, {}) @@ -37,6 +38,12 @@ def ready(self): else: self._validate_legacy_config(plugin_config) + # Auto-create the librenms_id custom field after migrations complete + post_migrate.connect( + _ensure_librenms_id_custom_field, + dispatch_uid="netbox_librenms_plugin_ensure_cf", + ) + def _validate_multi_server_config(self, servers_config): """Validate multi-server configuration.""" if not servers_config or not isinstance(servers_config, dict): @@ -61,4 +68,72 @@ def _validate_legacy_config(self, plugin_config): ) +def _ensure_librenms_id_custom_field(sender, **kwargs): + """ + Auto-create (or migrate) the 'librenms_id' custom field. + Runs after migrations via post_migrate signal to ensure tables exist. + Uses dispatch_uid to avoid duplicate connections. + + librenms_id stores a per-server JSON mapping {"server_key": device_id}. + Legacy installations may have this field typed as 'integer'; we upgrade it + to 'json' automatically so the UI and API accept the dict format. + """ + # Only run once per migrate invocation (post_migrate fires per-app). + if getattr(_ensure_librenms_id_custom_field, "_executed", False): + return + + import logging + + try: + from django.contrib.contenttypes.models import ContentType + + from extras.models import CustomField + + cf, created = CustomField.objects.get_or_create( + name="librenms_id", + defaults={ + "type": "json", + "label": "LibreNMS ID", + "description": "LibreNMS Device ID for synchronization (auto-created by plugin)", + "required": False, + "ui_visible": "if-set", + "ui_editable": "yes", + "is_cloneable": False, + }, + ) + + # Migrate legacy integer-typed field to JSON so the multi-server + # dict format {"server_key": device_id} is accepted by the UI/API. + if not created and cf.type == "integer": + cf.type = "json" + cf.save(update_fields=["type"]) + logging.getLogger("netbox_librenms_plugin").info( + "Migrated 'librenms_id' custom field type from integer to json" + ) + + # Ensure the field is assigned to the required object types + from dcim.models import Device, Interface + from virtualization.models import VirtualMachine, VMInterface + + required_models = [Device, VirtualMachine, Interface, VMInterface] + current_types = set(cf.object_types.values_list("pk", flat=True)) + + for model in required_models: + ct = ContentType.objects.get_for_model(model) + if ct.pk not in current_types: + cf.object_types.add(ct) + + if created: + logging.getLogger("netbox_librenms_plugin").info( + "Auto-created 'librenms_id' custom field for Device, VirtualMachine, Interface, VMInterface" + ) + + # Only mark as executed after successful completion to allow retry on failure. + _ensure_librenms_id_custom_field._executed = True + except Exception as e: + # Don't break startup if custom field creation fails (e.g., during initial migration), + # but log the error so it's not silently swallowed. + logging.getLogger("netbox_librenms_plugin").exception("Failed to auto-create 'librenms_id' custom field: %s", e) + + config = LibreNMSSyncConfig diff --git a/netbox_librenms_plugin/api/serializers.py b/netbox_librenms_plugin/api/serializers.py index 6bcd0aef20..bcde788d2b 100644 --- a/netbox_librenms_plugin/api/serializers.py +++ b/netbox_librenms_plugin/api/serializers.py @@ -1,6 +1,12 @@ from netbox.api.serializers import NetBoxModelSerializer -from netbox_librenms_plugin.models import InterfaceTypeMapping +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) class InterfaceTypeMappingSerializer(NetBoxModelSerializer): @@ -11,3 +17,51 @@ class Meta: model = InterfaceTypeMapping fields = ["id", "librenms_type", "librenms_speed", "netbox_type", "description"] + + +class DeviceTypeMappingSerializer(NetBoxModelSerializer): + """Serialize DeviceTypeMapping model for REST API.""" + + class Meta: + """Meta options for DeviceTypeMappingSerializer.""" + + model = DeviceTypeMapping + fields = ["id", "librenms_hardware", "netbox_device_type", "description"] + + +class ModuleTypeMappingSerializer(NetBoxModelSerializer): + """Serialize ModuleTypeMapping model for REST API.""" + + class Meta: + """Meta options for ModuleTypeMappingSerializer.""" + + model = ModuleTypeMapping + fields = ["id", "librenms_model", "netbox_module_type", "description"] + + +class ModuleBayMappingSerializer(NetBoxModelSerializer): + """Serialize ModuleBayMapping model for REST API.""" + + class Meta: + """Meta options for ModuleBayMappingSerializer.""" + + model = ModuleBayMapping + fields = ["id", "librenms_name", "librenms_class", "netbox_bay_name", "is_regex", "description"] + + +class NormalizationRuleSerializer(NetBoxModelSerializer): + """Serialize NormalizationRule model for REST API.""" + + class Meta: + """Meta options for NormalizationRuleSerializer.""" + + model = NormalizationRule + fields = [ + "id", + "scope", + "manufacturer", + "match_pattern", + "replacement", + "priority", + "description", + ] diff --git a/netbox_librenms_plugin/api/urls.py b/netbox_librenms_plugin/api/urls.py index 230aa078d0..c032e7b2f5 100644 --- a/netbox_librenms_plugin/api/urls.py +++ b/netbox_librenms_plugin/api/urls.py @@ -7,6 +7,10 @@ router = NetBoxRouter() router.register("interface-type-mappings", views.InterfaceTypeMappingViewSet) +router.register("device-type-mappings", views.DeviceTypeMappingViewSet) +router.register("module-type-mappings", views.ModuleTypeMappingViewSet) +router.register("module-bay-mappings", views.ModuleBayMappingViewSet) +router.register("normalization-rules", views.NormalizationRuleViewSet) urlpatterns = [ path("jobs//sync-status/", views.sync_job_status, name="sync_job_status"), diff --git a/netbox_librenms_plugin/api/views.py b/netbox_librenms_plugin/api/views.py index 768c67f5fe..287a3858c1 100644 --- a/netbox_librenms_plugin/api/views.py +++ b/netbox_librenms_plugin/api/views.py @@ -11,9 +11,21 @@ from rq.job import Job as RQJob from netbox_librenms_plugin.constants import PERM_CHANGE_PLUGIN, PERM_VIEW_PLUGIN -from netbox_librenms_plugin.models import InterfaceTypeMapping - -from .serializers import InterfaceTypeMappingSerializer +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) + +from .serializers import ( + DeviceTypeMappingSerializer, + InterfaceTypeMappingSerializer, + ModuleBayMappingSerializer, + ModuleTypeMappingSerializer, + NormalizationRuleSerializer, +) logger = logging.getLogger(__name__) @@ -22,8 +34,8 @@ class LibreNMSPluginPermission(BasePermission): """ Permission class for LibreNMS plugin API endpoints. - - GET requests require view_librenmssettings - - All other requests require change_librenmssettings + - Safe requests (GET, HEAD, OPTIONS) require netbox_librenms_plugin.view_librenmssettings + - All other requests require netbox_librenms_plugin.change_librenmssettings """ def has_permission(self, request, view): @@ -41,6 +53,42 @@ class InterfaceTypeMappingViewSet(NetBoxModelViewSet): serializer_class = InterfaceTypeMappingSerializer +class DeviceTypeMappingViewSet(NetBoxModelViewSet): + """API viewset for DeviceTypeMapping CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = DeviceTypeMapping.objects.all() + serializer_class = DeviceTypeMappingSerializer + + +class ModuleTypeMappingViewSet(NetBoxModelViewSet): + """API viewset for ModuleTypeMapping CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = ModuleTypeMapping.objects.all() + serializer_class = ModuleTypeMappingSerializer + + +class ModuleBayMappingViewSet(NetBoxModelViewSet): + """API viewset for ModuleBayMapping CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = ModuleBayMapping.objects.all() + serializer_class = ModuleBayMappingSerializer + + +class NormalizationRuleViewSet(NetBoxModelViewSet): + """API viewset for NormalizationRule CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = NormalizationRule.objects.all() + serializer_class = NormalizationRuleSerializer + + @api_view(["POST"]) @permission_classes([LibreNMSPluginPermission]) def sync_job_status(request, job_pk): diff --git a/netbox_librenms_plugin/filters.py b/netbox_librenms_plugin/filters.py index 9ec162a64c..134bd8962d 100644 --- a/netbox_librenms_plugin/filters.py +++ b/netbox_librenms_plugin/filters.py @@ -1,6 +1,6 @@ import django_filters -from .models import InterfaceTypeMapping +from .models import DeviceTypeMapping, InterfaceTypeMapping, ModuleBayMapping, ModuleTypeMapping, NormalizationRule class InterfaceTypeMappingFilterSet(django_filters.FilterSet): @@ -11,3 +11,43 @@ class Meta: model = InterfaceTypeMapping fields = ["librenms_type", "librenms_speed", "netbox_type", "description"] + + +class DeviceTypeMappingFilterSet(django_filters.FilterSet): + """Filter set for DeviceTypeMapping model.""" + + class Meta: + """Meta options for DeviceTypeMappingFilterSet.""" + + model = DeviceTypeMapping + fields = ["librenms_hardware", "description"] + + +class ModuleTypeMappingFilterSet(django_filters.FilterSet): + """Filter set for ModuleTypeMapping model.""" + + class Meta: + """Meta options for ModuleTypeMappingFilterSet.""" + + model = ModuleTypeMapping + fields = ["librenms_model", "description"] + + +class ModuleBayMappingFilterSet(django_filters.FilterSet): + """Filter set for ModuleBayMapping model.""" + + class Meta: + """Meta options for ModuleBayMappingFilterSet.""" + + model = ModuleBayMapping + fields = ["librenms_name", "librenms_class", "netbox_bay_name", "is_regex"] + + +class NormalizationRuleFilterSet(django_filters.FilterSet): + """Filter set for NormalizationRule model.""" + + class Meta: + """Meta options for NormalizationRuleFilterSet.""" + + model = NormalizationRule + fields = ["scope", "manufacturer"] diff --git a/netbox_librenms_plugin/forms.py b/netbox_librenms_plugin/forms.py index f3e3d075e5..9c0bc7d9dc 100644 --- a/netbox_librenms_plugin/forms.py +++ b/netbox_librenms_plugin/forms.py @@ -2,7 +2,7 @@ import logging from dcim.choices import InterfaceTypeChoices -from dcim.models import Device, DeviceRole, DeviceType, Location, Rack, Site +from dcim.models import Device, DeviceRole, DeviceType, Location, Manufacturer, ModuleType, Rack, Site from django import forms from django.http import QueryDict from django.utils.translation import gettext_lazy as _ @@ -12,10 +12,22 @@ NetBoxModelImportForm, ) from netbox.plugins import get_plugin_config -from utilities.forms.fields import CSVChoiceField, DynamicModelMultipleChoiceField +from utilities.forms.fields import ( + CSVChoiceField, + CSVModelChoiceField, + DynamicModelChoiceField, + DynamicModelMultipleChoiceField, +) from virtualization.models import Cluster, VirtualMachine -from .models import InterfaceTypeMapping, LibreNMSSettings +from .models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + LibreNMSSettings, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) logger = logging.getLogger(__name__) @@ -51,11 +63,24 @@ def _get_librenms_poller_group_choices(): """ Helper function to get poller group choices from LibreNMS API. Shared between AddToLIbreSNMPV1V2 and AddToLIbreSNMPV3 forms. + Results are cached to avoid repeated API calls on every form instantiation. """ + from django.core.cache import cache + from .librenms_api import LibreNMSAPI choices = [("0", "Default (0)")] + try: + api = LibreNMSAPI() + server_id = api.librenms_url.rstrip("/") + cache_key = f"librenms_poller_group_choices_{server_id}" + except Exception: + cache_key = "librenms_poller_group_choices" + cached_choices = cache.get(cache_key) + if cached_choices: + return cached_choices + try: api = LibreNMSAPI() success, poller_groups = api.get_poller_groups() @@ -72,6 +97,8 @@ def _get_librenms_poller_group_choices(): else: label = f"{group_name} ({group_id})" choices.append((group_id, label)) + + cache.set(cache_key, choices, timeout=api.cache_timeout) except Exception: logger.exception("Failed to fetch LibreNMS poller groups; using default choices") @@ -90,10 +117,13 @@ class ServerConfigForm(NetBoxModelForm): ) class Meta: + """Meta options for ServerConfigForm.""" + model = LibreNMSSettings fields = ["selected_server"] def __init__(self, *args, **kwargs): + """Initialize form and populate server choices.""" super().__init__(*args, **kwargs) self.fields["selected_server"].choices = _get_librenms_server_choices() @@ -131,6 +161,8 @@ class ImportSettingsForm(NetBoxModelForm): ) class Meta: + """Meta options for ImportSettingsForm.""" + model = LibreNMSSettings fields = [ "vc_member_name_pattern", @@ -213,6 +245,8 @@ class InterfaceTypeMappingForm(NetBoxModelForm): """ class Meta: + """Meta options for InterfaceTypeMappingForm.""" + model = InterfaceTypeMapping fields = ["librenms_type", "librenms_speed", "netbox_type", "description"] @@ -230,6 +264,8 @@ class InterfaceTypeMappingImportForm(NetBoxModelImportForm): ) class Meta: + """Meta options for InterfaceTypeMappingImportForm.""" + model = InterfaceTypeMapping fields = ["librenms_type", "librenms_speed", "netbox_type", "description"] @@ -260,6 +296,175 @@ class InterfaceTypeMappingFilterForm(NetBoxModelFilterSetForm): model = InterfaceTypeMapping +class DeviceTypeMappingForm(NetBoxModelForm): + """Form for creating and editing device type mappings between LibreNMS and NetBox.""" + + netbox_device_type = forms.ModelChoiceField( + queryset=DeviceType.objects.all(), + label="NetBox Device Type", + widget=forms.Select(attrs={"class": "form-select"}), + ) + + class Meta: + """Meta options for DeviceTypeMappingForm.""" + + model = DeviceTypeMapping + fields = ["librenms_hardware", "netbox_device_type", "description"] + + +class DeviceTypeMappingImportForm(NetBoxModelImportForm): + """Form for bulk importing device type mappings.""" + + netbox_device_type = CSVModelChoiceField( + queryset=DeviceType.objects.all(), + to_field_name="model", + help_text="NetBox device type model name", + ) + + class Meta: + """Meta options for DeviceTypeMappingImportForm.""" + + model = DeviceTypeMapping + fields = ["librenms_hardware", "netbox_device_type", "description"] + + +class DeviceTypeMappingFilterForm(NetBoxModelFilterSetForm): + """Form for filtering device type mappings.""" + + librenms_hardware = forms.CharField(required=False, label="LibreNMS Hardware") + description = forms.CharField( + required=False, + label="Description", + help_text="Filter by description (partial match)", + ) + + model = DeviceTypeMapping + + +class ModuleTypeMappingForm(NetBoxModelForm): + """Form for creating and editing module type mappings between LibreNMS and NetBox.""" + + class Meta: + """Meta options for ModuleTypeMappingForm.""" + + model = ModuleTypeMapping + fields = ["librenms_model", "netbox_module_type", "description"] + + +class ModuleTypeMappingImportForm(NetBoxModelImportForm): + """Form for bulk importing module type mappings.""" + + netbox_module_type = CSVModelChoiceField( + queryset=ModuleType.objects.all(), + to_field_name="model", + help_text="NetBox module type model name", + ) + + class Meta: + """Meta options for ModuleTypeMappingImportForm.""" + + model = ModuleTypeMapping + fields = ["librenms_model", "netbox_module_type", "description"] + + +class ModuleTypeMappingFilterForm(NetBoxModelFilterSetForm): + """Form for filtering module type mappings.""" + + librenms_model = forms.CharField(required=False, label="LibreNMS Model") + description = forms.CharField( + required=False, + label="Description", + help_text="Filter by description (partial match)", + ) + + model = ModuleTypeMapping + + +class ModuleBayMappingForm(NetBoxModelForm): + """Form for creating and editing module bay mappings between LibreNMS and NetBox.""" + + class Meta: + """Meta options for ModuleBayMappingForm.""" + + model = ModuleBayMapping + fields = ["librenms_name", "librenms_class", "netbox_bay_name", "is_regex", "description"] + + +class ModuleBayMappingImportForm(NetBoxModelImportForm): + """Form for bulk importing module bay mappings.""" + + class Meta: + """Meta options for ModuleBayMappingImportForm.""" + + model = ModuleBayMapping + fields = ["librenms_name", "librenms_class", "netbox_bay_name", "is_regex", "description"] + + +class ModuleBayMappingFilterForm(NetBoxModelFilterSetForm): + """Form for filtering module bay mappings.""" + + librenms_name = forms.CharField(required=False, label="LibreNMS Name") + librenms_class = forms.CharField(required=False, label="LibreNMS Class") + netbox_bay_name = forms.CharField(required=False, label="NetBox Bay Name") + is_regex = forms.NullBooleanField(required=False, label="Regex") + + model = ModuleBayMapping + + +class NormalizationRuleForm(NetBoxModelForm): + """Form for creating and editing normalization rules.""" + + manufacturer = DynamicModelChoiceField( + queryset=Manufacturer.objects.all(), + required=False, + help_text="Optional: scope this rule to a specific manufacturer", + ) + + class Meta: + """Meta options for NormalizationRuleForm.""" + + model = NormalizationRule + fields = ["scope", "manufacturer", "match_pattern", "replacement", "priority", "description"] + + +class NormalizationRuleImportForm(NetBoxModelImportForm): + """Form for bulk importing normalization rules.""" + + scope = CSVChoiceField( + choices=NormalizationRule.SCOPE_CHOICES, + help_text="Scope: module_type, device_type, or module_bay", + ) + manufacturer = CSVModelChoiceField( + queryset=Manufacturer.objects.all(), + to_field_name="name", + required=False, + help_text="Optional manufacturer name (must already exist in NetBox)", + ) + + class Meta: + """Meta options for NormalizationRuleImportForm.""" + + model = NormalizationRule + fields = ["scope", "manufacturer", "match_pattern", "replacement", "priority", "description"] + + +class NormalizationRuleFilterForm(NetBoxModelFilterSetForm): + """Form for filtering normalization rules.""" + + scope = forms.ChoiceField( + required=False, + choices=[("", "---------")] + NormalizationRule.SCOPE_CHOICES, + label="Scope", + ) + manufacturer_id = DynamicModelChoiceField( + queryset=Manufacturer.objects.all(), + required=False, + label="Manufacturer", + ) + + model = NormalizationRule + + class AddToLIbreSNMPV1V2(forms.Form): """ Form for adding devices to LibreNMS using SNMPv1 or SNMPv2c authentication. @@ -315,6 +520,7 @@ class AddToLIbreSNMPV1V2(forms.Form): ) def __init__(self, *args, **kwargs): + """Initialize form and populate poller group choices.""" super().__init__(*args, **kwargs) self.fields["poller_group"].choices = _get_librenms_poller_group_choices() @@ -412,6 +618,7 @@ class AddToLIbreSNMPV3(forms.Form): ) def __init__(self, *args, **kwargs): + """Initialize form and populate poller group choices.""" super().__init__(*args, **kwargs) self.fields["poller_group"].choices = _get_librenms_poller_group_choices() @@ -422,6 +629,7 @@ class DeviceStatusFilterForm(NetBoxModelFilterSetForm): """ def __init__(self, *args, **kwargs): + """Initialize form and remove saved filter field.""" super().__init__(*args, **kwargs) # Remove the saved filter field if it exists if "filter_id" in self.fields: @@ -581,7 +789,8 @@ def _populate_librenms_locations(self): try: # Use caching to avoid repeated API calls - cache_key = "librenms_locations_choices" + api = LibreNMSAPI() + cache_key = f"librenms_locations_choices:{api.server_key}" cached_choices = cache.get(cache_key) if cached_choices: @@ -589,7 +798,6 @@ def _populate_librenms_locations(self): return # Fetch locations from LibreNMS - api = LibreNMSAPI() success, locations = api.get_locations() if success and locations: diff --git a/netbox_librenms_plugin/import_utils.py b/netbox_librenms_plugin/import_utils.py deleted file mode 100644 index cb9c9a99e7..0000000000 --- a/netbox_librenms_plugin/import_utils.py +++ /dev/null @@ -1,2334 +0,0 @@ -""" -Utilities for importing devices from LibreNMS to NetBox. - -This module provides functions for: -- Validating LibreNMS devices for import -- Retrieving filtered LibreNMS devices -- Importing single and multiple devices -- Smart matching of NetBox objects -- Permission checking for import operations -""" - -import logging -from typing import List - -from core.choices import JobStatusChoices -from dcim.models import Device, DeviceRole, DeviceType, Rack, Site, VirtualChassis -from django.core.cache import cache -from django.core.exceptions import PermissionDenied -from django.db import transaction -from django.utils import timezone -from virtualization.models import Cluster - -from .librenms_api import LibreNMSAPI -from .utils import ( - find_matching_platform, - find_matching_site, - match_librenms_hardware_to_device_type, -) - -logger = logging.getLogger(__name__) - - -# ============================================================================= -# Permission Check Helpers -# ============================================================================= - - -def check_user_permissions(user, permissions): - """ - Check if user has all required permissions. - - Args: - user: The user object to check permissions for - permissions: List of permission strings (e.g., ['dcim.add_device', 'dcim.add_interface']) - - Returns: - tuple: (has_all_permissions: bool, missing_permissions: list[str]) - - Raises: - PermissionDenied: If user is None (no user context available) - """ - if user is None: - raise PermissionDenied("No user context available for permission check") - - missing = [perm for perm in permissions if not user.has_perm(perm)] - return (len(missing) == 0, missing) - - -def require_permissions(user, permissions, action_description="perform this action"): - """ - Require user has all permissions, raising PermissionDenied if not. - - Args: - user: The user object to check permissions for - permissions: List of permission strings - action_description: Human-readable description for error message - - Raises: - PermissionDenied: If user lacks any required permission - """ - has_perms, missing = check_user_permissions(user, permissions) - if not has_perms: - missing_str = ", ".join(missing) - raise PermissionDenied( - f"You do not have permission to {action_description}. Missing permissions: {missing_str}" - ) - - -def get_cache_metadata_key(server_key: str, filters: dict, vc_enabled: bool) -> str: - """ - Generate a consistent cache metadata key from filter parameters. - - Args: - server_key: LibreNMS server identifier - filters: Filter dictionary - vc_enabled: Whether VC detection is enabled - - Returns: - str: Consistent cache key for metadata - """ - # Sort filter items to ensure consistent key generation - filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v) - return f"librenms_filter_cache_metadata_{server_key}_{filter_parts}_{vc_enabled}" - - -def get_active_cached_searches(server_key: str) -> list[dict]: - """ - Retrieve all active cached searches for a server and enrich with display-friendly values. - - Enriches raw filter IDs with human-readable names by looking up location names - from cached choices and converting type codes to display names. - - Args: - server_key: LibreNMS server identifier - - Returns: - List of dicts containing cache metadata with enriched display_filters - """ - from datetime import datetime, timezone - - cache_index_key = f"librenms_cache_index_{server_key}" - cache_index = cache.get(cache_index_key, []) - - active_searches = [] - valid_cache_keys = [] - - # Get location and type choices for enriching display - location_choices = {} - type_choices = { - "": "All Types", - "network": "Network", - "server": "Server", - "storage": "Storage", - "wireless": "Wireless", - "firewall": "Firewall", - "power": "Power", - "appliance": "Appliance", - "printer": "Printer", - "loadbalancer": "Load Balancer", - "other": "Other", - } - - # Get cached location choices for enrichment - location_cache_key = "librenms_locations_choices" - cached_locations = cache.get(location_cache_key) - if cached_locations: - location_choices = dict(cached_locations) - - for cache_key in cache_index: - metadata = cache.get(cache_key) - if metadata: - # Cache still exists, calculate time remaining - cached_at = datetime.fromisoformat(metadata.get("cached_at")) - cache_timeout = metadata.get("cache_timeout", 300) - now = datetime.now(timezone.utc) - age_seconds = (now - cached_at).total_seconds() - remaining_seconds = max(0, cache_timeout - age_seconds) - - if remaining_seconds > 0: - # Add remaining time and cache key - metadata["remaining_seconds"] = int(remaining_seconds) - metadata["cache_key"] = cache_key - - # Enrich filters with human-readable display values - if "filters" in metadata: - display_filters = metadata["filters"].copy() - # Convert location ID to location name - if "location" in display_filters and display_filters["location"] in location_choices: - display_filters["location"] = location_choices[display_filters["location"]] - # Convert type code to display name - if "type" in display_filters and display_filters["type"] in type_choices: - display_filters["type"] = type_choices[display_filters["type"]] - metadata["display_filters"] = display_filters - else: - # Fallback if filters key missing - metadata["display_filters"] = {} - - active_searches.append(metadata) - valid_cache_keys.append(cache_key) - - # Clean up index if any keys have expired - if len(valid_cache_keys) < len(cache_index): - cache.set(cache_index_key, valid_cache_keys, timeout=3600) - - # Sort by most recent first - active_searches.sort(key=lambda x: x.get("cached_at", ""), reverse=True) - - return active_searches - - -def get_validated_device_cache_key(server_key: str, filters: dict, device_id: int | str, vc_enabled: bool) -> str: - """ - Generate a consistent cache key for validated device data. - - This ensures both synchronous and background job processing use the same - cache keys, avoiding duplicate validation work and cache entries. - - Args: - server_key: LibreNMS server key - filters: Filter dict with location, type, os, hostname, sysname, hardware keys - device_id: LibreNMS device ID - vc_enabled: Whether virtual chassis detection was enabled - - Returns: - str: Cache key for the validated device - - Example: - >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) - >>> key - 'validated_device_default_-1234567890_123_vc' - """ - # Sort filters for consistent hashing - filter_hash = hash(str(sorted(filters.items()))) - vc_part = "vc" if vc_enabled else "novc" - return f"validated_device_{server_key}_{filter_hash}_{device_id}_{vc_part}" - - -def get_import_device_cache_key(device_id: int | str, server_key: str = "default") -> str: - """ - Generate cache key for raw LibreNMS device data. - - This key is used to cache raw device data (without validation metadata) - to avoid redundant API calls when users interact with dropdowns during - the import workflow. - - Args: - device_id: LibreNMS device ID - server_key: LibreNMS server identifier for multi-server setups - - Returns: - str: Cache key for the device data - - Example: - >>> get_import_device_cache_key(123, "production") - 'import_device_data_production_123' - """ - return f"import_device_data_{server_key}_{device_id}" - - -def _determine_device_name( - libre_device: dict, - use_sysname: bool = True, - strip_domain: bool = False, - device_id: int | str = None, -) -> str: - """ - Determine the device/VM name from LibreNMS data. - - Centralized logic for building device names with consistent handling of: - - sysName vs hostname preference - - Domain stripping (avoiding IP addresses) - - Fallback to device_id when name is missing - - Args: - libre_device: Device data from LibreNMS - use_sysname: If True, prefer sysName; if False, use hostname - strip_domain: If True, strip domain suffix (e.g., '.example.com') - device_id: LibreNMS device ID for fallback name generation - - Returns: - str: The determined device name - - Example: - >>> _determine_device_name({'sysName': 'router.example.com', 'hostname': 'router'}, - ... use_sysname=True, strip_domain=True) - 'router' - """ - # Determine base name based on use_sysname preference - if use_sysname: - name = libre_device.get("sysName") or libre_device.get("hostname") - else: - name = libre_device.get("hostname") or libre_device.get("sysName") - - # Fallback to device_id if no name found - if not name: - if device_id is not None: - name = f"device-{device_id}" - else: - name = libre_device.get("device_id", "unknown") - name = f"device-{name}" - - # Strip domain if requested (but not for IP addresses) - if strip_domain and name and "." in name: - try: - from ipaddress import ip_address - - ip_address(name) - # It's a valid IP address, don't strip - except ValueError: - # Not an IP, safe to strip domain - name = name.split(".")[0] - - return name - - -def empty_virtual_chassis_data() -> dict: - """Public helper for callers that need a blank VC payload.""" - - return { - "is_stack": False, - "member_count": 0, - "members": [], - "detection_error": None, - } - - -def _clone_virtual_chassis_data(data: dict | None) -> dict: - """Return a defensive copy of cached VC data to avoid shared references.""" - - if not data: - return empty_virtual_chassis_data() - - members = [] - for idx, member in enumerate(data.get("members", [])): - member_copy = member.copy() - raw_position = member_copy.get("position", idx) - try: - member_copy["position"] = int(raw_position) - except (TypeError, ValueError): - member_copy["position"] = idx - members.append(member_copy) - - member_count = data.get("member_count") or len(members) - - return { - "is_stack": bool(data.get("is_stack")), - "member_count": member_count, - "members": members, - "detection_error": data.get("detection_error"), - } - - -_VC_CACHE_VERSION = "v1" - - -def _vc_cache_key(api: LibreNMSAPI, device_id: int | str) -> str: - server_key = getattr(api, "server_key", "default") - return f"librenms_vc_detection_{_VC_CACHE_VERSION}_{server_key}_{device_id}" - - -def get_virtual_chassis_data(api: LibreNMSAPI, device_id: int | str, *, force_refresh: bool = False) -> dict: - """Fetch (and cache) virtual chassis data for a LibreNMS device.""" - - if not api or device_id is None: - return empty_virtual_chassis_data() - - cache_key = _vc_cache_key(api, device_id) - if not force_refresh: - cached = cache.get(cache_key) - if cached is not None: - return _clone_virtual_chassis_data(cached) - - detection_data = detect_virtual_chassis_from_inventory(api, device_id) - if detection_data and "detection_error" not in detection_data: - detection_data["detection_error"] = None - - cache_value = _clone_virtual_chassis_data(detection_data) if detection_data else empty_virtual_chassis_data() - - cache_timeout = getattr(api, "cache_timeout", 300) or 300 - cache.set(cache_key, cache_value, timeout=cache_timeout) - return _clone_virtual_chassis_data(cache_value) - - -def prefetch_vc_data_for_devices(api: LibreNMSAPI, device_ids: List[int], *, force_refresh: bool = False) -> None: - """ - Pre-warm the virtual chassis cache for multiple devices. - - This eliminates the 0.5-1s delay when rendering the import table - by proactively fetching VC data before validation. - - Args: - api: LibreNMSAPI instance - device_ids: List of LibreNMS device IDs to prefetch VC data for - force_refresh: When True, bypass cache and fetch fresh data - - Example: - >>> # Before rendering import table - >>> prefetch_vc_data_for_devices(api, [123, 124, 125]) - >>> # Now all validate_device_for_import() calls hit cache instantly - """ - if not api or not device_ids: - return - - logger.debug(f"Pre-warming VC cache for {len(device_ids)} devices") - - for idx, device_id in enumerate(device_ids): - # This populates the cache if empty, or skips if already cached - try: - get_virtual_chassis_data(api, device_id, force_refresh=force_refresh) - except (BrokenPipeError, ConnectionError, IOError, OSError) as e: - logger.warning(f"Connection error during VC prefetch at device {idx}: {e}") - # Stop processing if connection is broken - return - except Exception as e: - # Log but continue for other errors - logger.warning(f"Error prefetching VC data for device {device_id}: {e}") - - logger.debug(f"VC cache warming complete for {len(device_ids)} devices") - - -def get_device_count_for_filters( - api: LibreNMSAPI, - filters: dict, - clear_cache: bool = False, - show_disabled: bool = True, -) -> int: - """ - Get count of LibreNMS devices matching filters. - - This is a lightweight function to determine device count for background job - decision making. Uses the same caching as get_librenms_devices_for_import(). - - Args: - api: LibreNMS API client instance - filters: Filter dict with location, type, os, hostname, sysname keys - clear_cache: Whether to force cache refresh - show_disabled: Whether to include disabled devices - - Returns: - int: Count of devices matching filters - """ - devices = get_librenms_devices_for_import(api, filters=filters, force_refresh=clear_cache) - - # Filter out disabled devices if requested - if not show_disabled: - devices = [d for d in devices if d.get("status") == 1] - - return len(devices) - - -def get_librenms_devices_for_import( - api: LibreNMSAPI = None, - filters: dict = None, - server_key: str = None, - *, - force_refresh: bool = False, - return_cache_status: bool = False, -) -> List[dict] | tuple[List[dict], bool]: - """ - Retrieve LibreNMS devices based on filters. - - Args: - api: LibreNMSAPI instance (if not provided, creates one with server_key) - filters: Dict containing filter parameters: - - location: LibreNMS location/site filter - - type: Device type filter - - os: Operating system filter - - hostname: Hostname filter (partial match) - - sysname: System name filter (partial match) - - status: Device status filter (1=up, 0=down) - - disabled: Include disabled devices (0=active only, 1=all) - server_key: Key for specific server configuration (used if api not provided) - force_refresh: When True, bypass the cache and fetch fresh data - return_cache_status: When True, returns (devices, from_cache) tuple - - Returns: - List of device dictionaries from LibreNMS, or tuple of (devices, from_cache) - if return_cache_status is True. from_cache=True means data was loaded from - existing cache; from_cache=False means data was just fetched from LibreNMS. - """ - try: - # Use provided API instance or create a new one - if api is None: - api = LibreNMSAPI(server_key=server_key) - - # Build LibreNMS API filters using the type/query format - # LibreNMS API v0 expects ?type=X&query=Y format, not direct parameters - # NOTE: API only supports ONE type/query pair, so we'll use the most - # specific filter for the API and apply others client-side - api_filters = {} - client_filters = {} # Filters to apply after fetching from API - - if filters: - # Check for status filter first - it has special handling - if filters.get("status") is not None: - # Status filter uses special types that don't need query param - if filters["status"] == 1: - api_filters["type"] = "up" - elif filters["status"] == 0: - api_filters["type"] = "down" - - # Save ALL other filters for client-side filtering when status is used - if filters.get("location"): - client_filters["location"] = filters["location"] - if filters.get("type"): - client_filters["type"] = filters["type"] - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - else: - # Priority order for type/query filters: location > type > os > hostname > sysname - # Note: When sysname is combined with other filters, it's applied client-side for partial matching - # When sysname is alone, it uses API exact match (type=sysName) - # Note: hardware is always applied client-side for partial matching - # Use first available for API, save others for client-side filtering - if filters.get("location"): - api_filters["type"] = "location_id" - api_filters["query"] = filters["location"] - # Save remaining filters for client-side - if filters.get("type"): - client_filters["type"] = filters["type"] - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("type"): - api_filters["type"] = "type" - api_filters["query"] = filters["type"] - # Save remaining filters for client-side - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("os"): - api_filters["type"] = "os" - api_filters["query"] = filters["os"] - # Save remaining filters for client-side - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("hostname"): - api_filters["type"] = "hostname" - api_filters["query"] = filters["hostname"] - # Save sysname and hardware for client-side - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("sysname"): - # sysname-only filter: Use API exact match (type=sysName&query=) - # This is safe - returns empty if no exact match found - api_filters["type"] = "sysName" - api_filters["query"] = filters["sysname"] - # Save hardware for client-side - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("hardware"): - # hardware-only filter: apply client-side for partial matching - client_filters["hardware"] = filters["hardware"] - - # Note: disabled filter isn't directly supported by LibreNMS API - # We'll filter client-side if needed - - # Use caching to avoid repeated API calls - # Include both API and client filters in cache key - cache_key = f"librenms_devices_import_{server_key}_{hash(str(api_filters))}_{hash(str(client_filters))}" - from_cache = False - - if force_refresh: - cache.delete(cache_key) - else: - cached_result = cache.get(cache_key) - if cached_result is not None: - # No need to deepcopy - cached data isn't mutated - devices = cached_result - from_cache = True - if return_cache_status: - return devices, from_cache - return devices - - success, devices = api.list_devices(api_filters if api_filters else None) - - if not success: - logger.error(f"Failed to retrieve devices from LibreNMS: {devices}") - if return_cache_status: - return [], False - return [] - - # Apply client-side filters if any - if client_filters: - devices = _apply_client_filters(devices, client_filters) - - # Cache using configured timeout (default 300s) - # No need to deepcopy - Django's cache backend handles serialization - cache.set(cache_key, devices, timeout=api.cache_timeout) - - if return_cache_status: - return devices, from_cache - return devices - - except Exception: - logger.exception("Error retrieving LibreNMS devices for import") - if return_cache_status: - return [], False - return [] - - -def _apply_client_filters(devices: List[dict], filters: dict) -> List[dict]: - """ - Apply client-side filters to device list. - - Args: - devices: List of device dicts from LibreNMS - filters: Dict of filters to apply (location, type, os, hostname, sysname) - - Returns: - Filtered list of devices - """ - filtered = devices - - if filters.get("location"): - location_id = str(filters["location"]) - filtered = [d for d in filtered if str(d.get("location_id", "")) == location_id] - - if filters.get("type"): - device_type = filters["type"].lower() - filtered = [d for d in filtered if d.get("type", "").lower() == device_type] - - if filters.get("os"): - os_filter = filters["os"].lower() - filtered = [d for d in filtered if os_filter in d.get("os", "").lower()] - - if filters.get("hostname"): - hostname_filter = filters["hostname"].lower() - filtered = [d for d in filtered if hostname_filter in d.get("hostname", "").lower()] - - if filters.get("sysname"): - sysname_filter = filters["sysname"].lower() - filtered = [d for d in filtered if sysname_filter in d.get("sysName", "").lower()] - - if filters.get("hardware"): - hardware_filter = filters["hardware"].lower() - filtered = [d for d in filtered if hardware_filter in (d.get("hardware") or "").lower()] - - return filtered - - -def validate_device_for_import( - libre_device: dict, - import_as_vm: bool = False, - api: "LibreNMSAPI" = None, - *, - include_vc_detection: bool = True, - force_vc_refresh: bool = False, -) -> dict: - """ - Validate if a LibreNMS device can be imported to NetBox. - - Performs comprehensive validation: - - Checks if device already exists in NetBox - - Validates required prerequisites (Site, DeviceType, DeviceRole for devices) - OR (Cluster for VMs) - - Provides smart matching for missing objects - - Detects virtual chassis/stack configuration (if API provided) - - Returns detailed validation status - - Args: - libre_device: Device data from LibreNMS - import_as_vm: If True, validate for VM import instead of device import - api: Optional LibreNMSAPI instance for virtual chassis detection - include_vc_detection: Skip VC detection when False to speed up bulk operations - force_vc_refresh: When True, bypass cached VC data and re-query LibreNMS - - Returns: - dict: Validation result with structure: - { - 'is_ready': bool, # Can import without user intervention - 'can_import': bool, # Can import (possibly after configuration) - 'import_as_vm': bool, # Whether importing as VM - 'existing_device': Device or VirtualMachine or None, - 'issues': List[str], # Blocking issues - 'warnings': List[str], # Non-blocking warnings - 'site': { # Only for devices - 'found': bool, - 'site': Site or None, - 'match_type': str, # 'exact' or None - 'suggestions': List[Site] # Alternative suggestions - }, - 'device_type': { # Only for devices - 'found': bool, - 'device_type': DeviceType or None, - 'match_type': str, # 'exact' or None - 'suggestions': List[dict] # Device types for user selection - }, - 'device_role': { # Only for devices - 'found': bool, # Always False - requires manual selection - 'role': DeviceRole or None, - 'available_roles': List[DeviceRole] # All roles for user selection - }, - 'cluster': { # Only for VMs - 'found': bool, # Always False - requires manual selection - 'cluster': Cluster or None, - 'available_clusters': List[Cluster] # All clusters for user selection - }, - 'platform': { - 'found': bool, - 'platform': Platform or None, - 'match_type': str # 'exact' or None - } - } - - Example: - >>> validation = validate_device_for_import(libre_device) - >>> if validation['is_ready']: - ... import_single_device(libre_device['device_id']) - """ - result = { - "is_ready": False, - "can_import": False, - "import_as_vm": import_as_vm, - "existing_device": None, - "existing_match_type": None, # Track how existing device was matched - "issues": [], - "warnings": [], - "virtual_chassis": empty_virtual_chassis_data(), - "site": { - "found": False, - "site": None, - "match_type": None, - "suggestions": [], - }, - "device_type": { - "found": False, - "device_type": None, - "match_type": None, - "suggestions": [], - }, - "device_role": { - "found": False, - "role": None, - "available_roles": [], - }, - "cluster": { - "found": False, - "cluster": None, - "available_clusters": [], - }, - "platform": {"found": False, "platform": None, "match_type": None}, - "rack": { - "found": False, - "rack": None, - "available_racks": [], - }, - } - - try: - # 1. Check if device/VM already exists in NetBox - # Always check both Devices AND VMs to properly detect existing objects - librenms_id = libre_device.get("device_id") - hostname = libre_device.get("hostname", "") - logger.debug( - f"Checking for existing device/VM: " - f"librenms_id={librenms_id} (type={type(librenms_id).__name__}), " - f"hostname={hostname}" - ) - - from virtualization.models import VirtualMachine - - # Check for existing VM first (by librenms_id custom field) - # Always query with int to match custom field type - try: - existing_vm = VirtualMachine.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() - except (ValueError, TypeError): - # librenms_id is not convertible to int; no match will be found - existing_vm = None - - if existing_vm: - logger.info(f"Found existing VM: {existing_vm.name} (matched by librenms_id={librenms_id})") - result["existing_device"] = existing_vm - result["existing_match_type"] = "librenms_id" - result["import_as_vm"] = True # Force VM mode since VM exists - result["warnings"].append(f"VM already imported to NetBox as '{existing_vm.name}'") - result["can_import"] = False - return result - - # Check for existing Device (by librenms_id custom field) - # Always query with int to match custom field type - try: - existing_device = Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() - except (ValueError, TypeError): - # librenms_id is not convertible to int; no match will be found - existing_device = None - - if existing_device: - logger.info(f"Found existing device: {existing_device.name} (matched by librenms_id={librenms_id})") - result["existing_device"] = existing_device - result["existing_match_type"] = "librenms_id" - result["warnings"].append(f"Device already imported to NetBox as '{existing_device.name}'") - result["can_import"] = False - return result - - # Check by hostname/name - Check both VMs and Devices for conflicts - existing_vm = VirtualMachine.objects.filter(name__iexact=hostname).first() - existing_device = Device.objects.filter(name__iexact=hostname).first() - - # If BOTH exist with same hostname, it's ambiguous - don't match either - if existing_vm and existing_device: - logger.warning( - f"Hostname conflict: Both VM '{existing_vm.name}' and Device " - f"'{existing_device.name}' exist with hostname '{hostname}'" - ) - result["warnings"].append( - f"Both a VM and Device exist with hostname '{hostname}' in NetBox. " - f"Cannot determine which to match. Please set the librenms_id custom field on the correct object." - ) - # Don't set existing_device, don't block import - let user proceed as new - # This allows them to import and then resolve the conflict manually - elif existing_vm: - logger.info(f"Found existing VM by hostname: {existing_vm.name}") - result["existing_device"] = existing_vm - result["existing_match_type"] = "hostname" - result["import_as_vm"] = True # Force VM mode since VM exists - result["warnings"].append( - f"VM with same hostname exists in NetBox as '{existing_vm.name}' (not linked to LibreNMS)" - ) - result["can_import"] = False - return result - elif existing_device: - logger.info(f"Found existing device by hostname: {existing_device.name}") - result["existing_device"] = existing_device - result["existing_match_type"] = "hostname" - result["warnings"].append( - f"Device with same hostname exists in NetBox as '{existing_device.name}' (not linked to LibreNMS)" - ) - result["can_import"] = False - return result - - # Check by primary IP (weaker match, IP could be reassigned) - only for devices - primary_ip = libre_device.get("ip") - if primary_ip and not import_as_vm: - from ipam.models import IPAddress - - existing_ip = IPAddress.objects.filter(address__startswith=primary_ip).first() - if existing_ip and existing_ip.assigned_object: - device = existing_ip.assigned_object.device if hasattr(existing_ip.assigned_object, "device") else None - if device: - result["existing_device"] = device - result["existing_match_type"] = "primary_ip" - result["warnings"].append( - f"IP address {primary_ip} already assigned to device '{device.name}' (not linked to LibreNMS)" - ) - result["can_import"] = False - return result - - # Validate based on import type (Device or VM) - if import_as_vm: - # 2. For VMs: Validate Cluster (required) - Must be manually selected - from virtualization.models import Cluster - - result["cluster"]["found"] = False - result["issues"].append("Cluster must be manually selected before importing as VM") - # Provide list of available clusters for user selection (cached) - cache_key = "librenms_import_all_clusters" - all_clusters = cache.get(cache_key) - if all_clusters is None: - all_clusters = list(Cluster.objects.all()) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, all_clusters, cache_timeout) - result["cluster"]["available_clusters"] = all_clusters - - # Skip device-specific validations for VMs - result["site"]["found"] = True # Not required for VMs - result["device_type"]["found"] = True # Not required for VMs - result["device_role"]["found"] = True # Not required for VMs - - else: - # 2. For Devices: Validate Site (required) - location = libre_device.get("location", "") - site_match = find_matching_site(location) - result["site"] = site_match - - if not site_match["found"]: - result["issues"].append(f"No matching site found for location: '{location}'") - # Get alternative suggestions - if location: - all_sites = Site.objects.all()[:10] # Limit for performance - result["site"]["suggestions"] = list(all_sites) - - # 3. Validate DeviceType (required) - hardware = libre_device.get("hardware", "") - dt_match = match_librenms_hardware_to_device_type(hardware) - result["device_type"] = dt_match - - if not dt_match["matched"]: - result["issues"].append(f"No matching device type found for hardware: '{hardware}'") - # Get some device types for user to choose from - all_device_types = DeviceType.objects.all()[:10] - result["device_type"]["suggestions"] = [ - { - "device_type": dt, - "similarity": 0.0, # No fuzzy matching, just showing options - "match_field": None, - } - for dt in all_device_types - ] - else: - # Rename 'matched' to 'found' for consistency - result["device_type"]["found"] = dt_match["matched"] - result["device_type"]["device_type"] = dt_match["device_type"] - result["device_type"]["match_type"] = dt_match["match_type"] - - # 4. DeviceRole (required) - Must be manually selected by user - logger.debug(f"[{hostname}] Issues BEFORE adding role issue: {result['issues']}") - result["device_role"]["found"] = False - result["issues"].append("Device role must be manually selected before import") - logger.debug(f"[{hostname}] Issues AFTER adding role issue: {result['issues']}") - # Provide list of available roles for user selection (cached) - cache_key = "librenms_import_all_roles" - all_roles = cache.get(cache_key) - if all_roles is None: - all_roles = list(DeviceRole.objects.all()) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, all_roles, cache_timeout) - result["device_role"]["available_roles"] = all_roles - - # 4b. Rack (optional) - Provide available racks for the matched site - if site_match["found"] and site_match["site"]: - site = site_match["site"] - # Use cache to optimize rack lookups per site - cache_key = f"librenms_import_racks_site_{site.pk}" - available_racks = cache.get(cache_key) - - if available_racks is None: - from dcim.models import Rack - from django.db.models import Q - - # Query racks for this site - include both: - # 1. Racks assigned to locations within the site - # 2. Racks directly assigned to the site (without location) - available_racks = list( - Rack.objects.filter(Q(location__site=site) | Q(site=site)) - .select_related("location", "site") - .order_by("location__name", "name") - ) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, available_racks, cache_timeout) - - result["rack"]["available_racks"] = available_racks - # Rack is optional, don't add to issues - result["rack"]["found"] = True # Mark as "found" even if None (optional field) - - # Skip VM-specific validations for devices - result["cluster"]["found"] = True # Not required for devices - - # 5. Match Platform (optional - same for both devices and VMs) - os = libre_device.get("os", "") - platform_match = find_matching_platform(os) - result["platform"] = platform_match - - if not platform_match["found"] and os: - result["warnings"].append(f"No matching platform found for OS: '{os}'") - - # 6. Additional validations - if not hostname: - result["issues"].append("Device has no hostname") - - # Serial number check - serial = libre_device.get("serial", "") - if serial and serial != "-": - existing_serial = Device.objects.filter(serial=serial).first() - if existing_serial: - result["warnings"].append(f"Serial number {serial} already exists on device: {existing_serial.name}") - - # 7. Virtual chassis detection (only for devices, not VMs) - if include_vc_detection and not import_as_vm and api is not None: - device_id = libre_device.get("device_id") - if device_id: - try: - logger.debug(f"Calling get_virtual_chassis_data for device {device_id}") - vc_detection = get_virtual_chassis_data(api, device_id, force_refresh=force_vc_refresh) - logger.debug( - f"VC detection result: is_stack={vc_detection.get('is_stack')}, " - f"member_count={vc_detection.get('member_count')}, " - f"members={len(vc_detection.get('members', []))}" - ) - if vc_detection: - result["virtual_chassis"] = vc_detection - if vc_detection["is_stack"]: - logger.debug( - f"Virtual chassis CONFIRMED for device {hostname}: " - f"{vc_detection['member_count']} members" - ) - except Exception as e: - logger.exception(f"Exception during VC detection for device {hostname}: {e}") - result["virtual_chassis"]["detection_error"] = str(e) - else: - logger.debug(f"No device_id found for {hostname}") - - # 8. Determine if device/VM is ready to import - result["can_import"] = len(result["issues"]) == 0 - - if import_as_vm: - # For VMs: only cluster is required - result["is_ready"] = result["can_import"] and result["cluster"]["found"] - else: - # For Devices: site, device_type, and device_role are required - result["is_ready"] = ( - result["can_import"] - and result["site"]["found"] - and result["device_type"]["found"] - and result["device_role"]["found"] - ) - - logger.debug( - f"Validation for {libre_device.get('hostname')} ({'VM' if import_as_vm else 'Device'}): " - f"issues={len(result['issues'])}, can_import={result['can_import']}, " - f"issues_list={result['issues']}" - ) - - return result - - except Exception as e: - logger.exception(f"Error validating device for import: {libre_device.get('hostname', 'unknown')}") - result["issues"].append(f"Validation error: {str(e)}") - return result - - -def import_single_device( - device_id: int, - server_key: str = None, - validation: dict = None, - manual_mappings: dict = None, - sync_options: dict = None, - libre_device: dict = None, -) -> dict: - """ - Import a single LibreNMS device to NetBox. - - Args: - device_id: LibreNMS device ID - server_key: LibreNMS server configuration key - validation: Pre-computed validation dict (optional) - manual_mappings: Manual object mappings (optional): - - site_id: NetBox Site ID - - device_type_id: NetBox DeviceType ID - - device_role_id: NetBox DeviceRole ID - - platform_id: NetBox Platform ID (optional) - - rack_id: NetBox Rack ID (optional) - sync_options: Sync options (optional): - - sync_interfaces: bool (default True) - - sync_cables: bool (default True) - - sync_ips: bool (default True) - - sync_fields: bool (default True) - libre_device: Pre-fetched LibreNMS device data (optional). - If provided, skips API call to fetch device info. - - Returns: - dict: Import result with structure: - { - 'success': bool, - 'device': Device object or None, - 'message': str, - 'error': str or None, - 'synced': { - 'interfaces': int, - 'cables': int, - 'ip_addresses': int - } - } - """ - try: - api = LibreNMSAPI(server_key=server_key) - - # Use pre-fetched device data if provided, otherwise fetch from API - if libre_device is None: - success, libre_device = api.get_device_info(device_id) - if not success or not libre_device: - return { - "success": False, - "device": None, - "message": "", - "error": f"Failed to retrieve device {device_id} from LibreNMS", - "synced": {}, - } - - # Validate device if validation not provided - if validation is None: - validation = validate_device_for_import(libre_device) - - # Check if device already exists - if validation.get("existing_device"): - return { - "success": False, - "device": validation["existing_device"], - "message": "", - "error": f"Device already exists: {validation['existing_device'].name}", - "synced": {}, - } - - # Use validation-derived matches, allow manual mappings to override specific fields - site = validation["site"].get("site") - device_type = validation["device_type"].get("device_type") - device_role = validation["device_role"].get("role") - platform = validation["platform"].get("platform") - rack = validation.get("rack", {}).get("rack") - - if manual_mappings: - site = Site.objects.filter(id=manual_mappings.get("site_id")).first() or site - device_type = DeviceType.objects.filter(id=manual_mappings.get("device_type_id")).first() or device_type - device_role = DeviceRole.objects.filter(id=manual_mappings.get("device_role_id")).first() or device_role - - platform_id = manual_mappings.get("platform_id") - if platform_id: - from dcim.models import Platform - - platform = Platform.objects.filter(id=platform_id).first() or platform - - rack_id = manual_mappings.get("rack_id") - if rack_id: - rack = Rack.objects.select_related("location", "site").filter(id=rack_id).first() or rack - - rack = rack or validation.get("rack", {}).get("rack") - - # Validate required fields - if not site: - return { - "success": False, - "device": None, - "message": "", - "error": "Site is required but not provided", - "synced": {}, - } - if not device_type: - return { - "success": False, - "device": None, - "message": "", - "error": "Device type is required but not provided", - "synced": {}, - } - if not device_role: - return { - "success": False, - "device": None, - "message": "", - "error": "Device role is required but not provided", - "synced": {}, - } - - # Create device in NetBox - with transaction.atomic(): - # Determine device name based on sync options - use_sysname = sync_options.get("use_sysname", True) if sync_options else True - strip_domain = sync_options.get("strip_domain", False) if sync_options else False - - device_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=strip_domain, - device_id=device_id, - ) - - # Generate import timestamp comment - import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") - - device_data = { - "name": device_name, - "site": site, - "device_type": device_type, - "role": device_role, - "status": "active" if libre_device.get("status") == 1 else "offline", - "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - "custom_field_data": {"librenms_id": int(device_id)}, - } - - # Add optional fields - if platform: - device_data["platform"] = platform - - if rack: - device_data["rack"] = rack - - serial = libre_device.get("serial", "") - if serial and serial != "-": - device_data["serial"] = serial - - location_name = libre_device.get("location", "") - if location_name and location_name != "-": - from dcim.models import Location - - # Try to find matching location within the site - location = Location.objects.filter(site=site, name__iexact=location_name).first() - if location: - device_data["location"] = location - - # Create the device - device = Device(**device_data) - device.full_clean() - device.save() - - # Sync additional data based on options - sync_options = sync_options or {} - synced = {"interfaces": 0, "cables": 0, "ip_addresses": 0} - - try: - # Sync interfaces - if sync_options.get("sync_interfaces", True): - # This is simplified - would need proper request context - # For now, just log that it should be done - logger.info(f"Interface sync should be performed for device {device.name}") - - # Sync cables - if sync_options.get("sync_cables", True): - logger.info(f"Cable sync should be performed for device {device.name}") - - # Sync IP addresses - if sync_options.get("sync_ips", True): - logger.info(f"IP address sync should be performed for device {device.name}") - - except Exception as e: - logger.warning(f"Error during post-import sync: {str(e)}") - # Don't fail the import if sync fails - - return { - "success": True, - "device": device, - "message": f"Successfully imported device: {device.name}", - "error": None, - "synced": synced, - } - - except Exception as e: - logger.exception(f"Error importing device {device_id}") - return { - "success": False, - "device": None, - "message": "", - "error": str(e), - "synced": {}, - } - - -def bulk_import_devices_shared( - device_ids: List[int], - server_key: str = None, - sync_options: dict = None, - manual_mappings_per_device: dict = None, - libre_devices_cache: dict = None, - job=None, - user=None, -) -> dict: - """ - Shared function for importing multiple LibreNMS devices to NetBox. - - Used by both synchronous imports and background jobs. Handles per-device error - collection and optional progress logging when job context is provided. - - Args: - device_ids: List of LibreNMS device IDs to import - server_key: LibreNMS server configuration key - sync_options: Sync options to apply to all devices - manual_mappings_per_device: Dict mapping device_id to manual_mappings dict - Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} - libre_devices_cache: Optional dict mapping device_id to pre-fetched device data - to avoid redundant API calls. Example: {123: {...device_data...}} - job: Optional JobRunner instance for progress logging and cancellation checks - user: User performing the import (for permission checks). If job is provided, - user is extracted from job.job.user if not explicitly passed. - - Returns: - dict: Bulk import result with structure: - { - 'total': int, - 'success': List[dict], # Successfully imported devices - 'failed': List[dict], # Failed imports with errors - 'skipped': List[dict], # Skipped devices (already exist, etc.) - 'virtual_chassis_created': int # Number of VCs created - } - - Raises: - PermissionDenied: If user lacks required permissions - - Example: - >>> # Synchronous usage - >>> result = bulk_import_devices_shared([1, 2, 3, 4, 5], user=request.user) - >>> # Background job usage - >>> result = bulk_import_devices_shared([1, 2, 3], job=self) - """ - # Extract user from job if not explicitly provided - if user is None and job is not None: - user = getattr(job.job, "user", None) - - # Check permissions at start of bulk operation - required_perms = [ - "dcim.add_device", - "dcim.add_interface", - "dcim.add_virtualchassis", - ] - require_permissions(user, required_perms, "import devices") - - total = len(device_ids) - success_list = [] - failed_list = [] - skipped_list = [] - vc_created_count = 0 - processed_vc_domains = set() # Track VCs already created by domain - - # Initialize API client once for all devices to avoid repeated config parsing - api = LibreNMSAPI(server_key=server_key) - - for idx, device_id in enumerate(device_ids, start=1): - # Check for job cancellation every 5 devices - if job and idx % 5 == 0: - # Refresh job from DB to get current status - job.job.refresh_from_db() - job_status = job.job.status - status_value = job_status.value if hasattr(job_status, "value") else job_status - if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): - if job.logger: - job.logger.warning(f"Import job cancelled at device {idx} of {total}") - else: - logger.warning(f"Import cancelled at device {idx} of {total}") - break - # Log progress - if job.logger: - job.logger.info(f"Imported device {idx} of {total}") - - try: - # Use cached device data if available to avoid redundant API calls - if libre_devices_cache and device_id in libre_devices_cache: - libre_device = libre_devices_cache[device_id] - success = True - else: - success, libre_device = api.get_device_info(device_id) - - if not success or not libre_device: - error_msg = f"Failed to retrieve device {device_id} from LibreNMS" - failed_list.append({"device_id": device_id, "error": error_msg}) - if job and job.logger: - job.logger.error(error_msg) - else: - logger.error(error_msg) - continue - - validation = validate_device_for_import(libre_device, api=api) - - # Build manual mappings from validation + any provided overrides - device_mappings = {} - - # Get site and device_type from validation - if validation["site"].get("found") and validation["site"].get("site"): - device_mappings["site_id"] = validation["site"]["site"].id - if validation["device_type"].get("found") and validation["device_type"].get("device_type"): - device_mappings["device_type_id"] = validation["device_type"]["device_type"].id - if validation["platform"].get("found") and validation["platform"].get("platform"): - device_mappings["platform_id"] = validation["platform"]["platform"].id - - # Override with any manual mappings provided for this device - if manual_mappings_per_device and device_id in manual_mappings_per_device: - device_mappings.update(manual_mappings_per_device[device_id]) - - result = import_single_device( - device_id, - server_key=server_key, - sync_options=sync_options, - manual_mappings=device_mappings if device_mappings else None, - libre_device=libre_device, - ) - - if result["success"]: - success_list.append( - { - "device_id": device_id, - "device": result["device"], - "message": result["message"], - } - ) - - # Handle virtual chassis creation for stacks - vc_data = validation.get("virtual_chassis", {}) - if vc_data.get("is_stack", False): - vc_domain = f"librenms-{device_id}" - - # Only create VC if we haven't processed this stack yet - # Add to set BEFORE attempting creation to prevent race condition - if vc_domain not in processed_vc_domains: - processed_vc_domains.add(vc_domain) - try: - vc = create_virtual_chassis_with_members( - result["device"], - vc_data["members"], - libre_device, - ) - vc_created_count += 1 - log_msg = f"Created VC '{vc.name}' during bulk import for device {device_id}" - if job and job.logger: - job.logger.info(log_msg) - else: - logger.info(log_msg) - except Exception as vc_error: - # Remove from set on failure so retry is possible - processed_vc_domains.discard(vc_domain) - warn_msg = f"Failed to create VC for device {device_id}: {vc_error}" - if job and job.logger: - job.logger.warning(warn_msg) - else: - logger.warning(warn_msg) - # Don't fail the import, just log the warning - - elif result.get("device"): # Device exists - skipped_list.append({"device_id": device_id, "reason": result["error"]}) - else: # Failed to import - failed_list.append({"device_id": device_id, "error": result["error"]}) - if job and job.logger: - job.logger.error(f"Failed to import device {device_id}: {result['error']}") - - except Exception as e: - error_msg = f"Unexpected error importing device {device_id}: {str(e)}" - if job and job.logger: - job.logger.error(error_msg, exc_info=True) - else: - logger.exception(f"Unexpected error importing device {device_id}") - failed_list.append({"device_id": device_id, "error": str(e)}) - - return { - "total": total, - "success": success_list, - "failed": failed_list, - "skipped": skipped_list, - "virtual_chassis_created": vc_created_count, - } - - -def bulk_import_devices( - device_ids: List[int], - server_key: str = None, - sync_options: dict = None, - manual_mappings_per_device: dict = None, - libre_devices_cache: dict = None, - user=None, -) -> dict: - """ - Import multiple LibreNMS devices to NetBox (synchronous). - - This is the public API for synchronous imports. For background job usage, - use bulk_import_devices_shared() with a job context. - - Args: - device_ids: List of LibreNMS device IDs to import - server_key: LibreNMS server configuration key - sync_options: Sync options to apply to all devices - manual_mappings_per_device: Dict mapping device_id to manual_mappings dict - Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} - libre_devices_cache: Optional dict mapping device_id to pre-fetched device data - to avoid redundant API calls. Example: {123: {...device_data...}} - user: User performing the import (for permission checks) - - Returns: - dict: Bulk import result with structure: - { - 'total': int, - 'success': List[dict], # Successfully imported devices - 'failed': List[dict], # Failed imports with errors - 'skipped': List[dict], # Skipped devices (already exist, etc.) - 'virtual_chassis_created': int # Number of VCs created - } - - Raises: - PermissionDenied: If user lacks required permissions - """ - return bulk_import_devices_shared( - device_ids=device_ids, - server_key=server_key, - sync_options=sync_options, - manual_mappings_per_device=manual_mappings_per_device, - libre_devices_cache=libre_devices_cache, - job=None, # No job context for synchronous imports - user=user, - ) - - -def get_librenms_device_by_id(api: LibreNMSAPI, device_id: int) -> dict: - """ - Retrieve a single device from LibreNMS by ID. - - Args: - api: LibreNMSAPI instance - device_id: LibreNMS device ID - - Returns: - Device dictionary or None if not found - """ - try: - # Use the dedicated API endpoint to get device by ID - success, device = api.get_device_info(device_id) - if success and device: - return device - - logger.warning(f"Device {device_id} not found in LibreNMS") - return None - except Exception as e: - logger.exception(f"Failed to get device {device_id} from LibreNMS: {e}") - return None - - -def fetch_device_with_cache( - device_id: int, - api: LibreNMSAPI, - server_key: str = None, - libre_devices_cache: dict = None, -) -> dict | None: - """ - Fetch LibreNMS device from cache or API with automatic caching. - - Checks three sources in order: - 1. Pre-fetched cache dict (if provided) - 2. Django cache (Redis/memory) - 3. LibreNMS API (caches result for future use) - - This function consolidates the device fetching pattern used throughout - the import workflow, eliminating code duplication. - - Args: - device_id: LibreNMS device ID to fetch - api: LibreNMSAPI instance for fallback API calls - server_key: Optional server key for multi-server setups (defaults to api.server_key) - libre_devices_cache: Optional pre-fetched device cache dict - - Returns: - Device dict from LibreNMS, or None if not found - - Example: - >>> # Simple usage - >>> libre_device = fetch_device_with_cache(123, api) - >>> if libre_device: - ... print(libre_device['hostname']) - >>> - >>> # With pre-fetched cache dict - >>> cache_dict = {123: {...}, 456: {...}} - >>> libre_device = fetch_device_with_cache(123, api, libre_devices_cache=cache_dict) - """ - # Check pre-fetched cache dict first (fastest) - if libre_devices_cache and device_id in libre_devices_cache: - return libre_devices_cache[device_id] - - # Check Django cache - cache_key = get_import_device_cache_key(device_id, server_key or api.server_key) - libre_device = cache.get(cache_key) - - if not libre_device: - # Fallback to API fetch - libre_device = get_librenms_device_by_id(api, device_id) - if libre_device: - # Cache for future use - cache.set(cache_key, libre_device, timeout=api.cache_timeout) - - return libre_device - - -def create_vm_from_librenms(libre_device: dict, validation: dict, use_sysname: bool = True, role=None): - """ - Create a NetBox VirtualMachine from LibreNMS device data. - - Args: - libre_device: Device data from LibreNMS - validation: Validation result from validate_device_for_import with import_as_vm=True - use_sysname: If True, prefer sysName; if False, use hostname - role: Optional DeviceRole to assign to the VM - - Returns: - Created VirtualMachine instance - - Raises: - Exception if VM cannot be created - """ - from virtualization.models import VirtualMachine - - if not validation["can_import"]: - raise ValueError(f"VM cannot be imported: {', '.join(validation['issues'])}") - - # Extract matched objects from validation - cluster = validation["cluster"]["cluster"] - platform = validation["platform"].get("platform") - - # Determine VM name - use pre-computed name if available (handles strip_domain) - vm_name = libre_device.get("_computed_name") - if not vm_name: - vm_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=False, - device_id=libre_device.get("device_id"), - ) - - # Generate import timestamp comment - import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") - - # Create the VM with librenms_id custom field - vm = VirtualMachine.objects.create( - name=vm_name, - cluster=cluster, - role=role, # Optional VM role - platform=platform, - comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - custom_field_data={"librenms_id": int(libre_device["device_id"])}, - ) - - logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") - return vm - - -def bulk_import_vms( - vm_imports: dict[int, dict[str, int]], - api: LibreNMSAPI, - sync_options: dict = None, - libre_devices_cache: dict = None, - job=None, - user=None, -) -> dict: - """ - Import multiple LibreNMS devices as VMs in NetBox. - - Handles validation, cluster/role assignment, name determination, - and VM creation. Supports both synchronous and background job execution. - - This function consolidates VM import logic that was previously duplicated - in BulkImportDevicesView and ImportDevicesJob, ensuring consistent behavior - across synchronous and background import paths. - - Args: - vm_imports: Dict mapping device_id to {"cluster_id": int, "device_role_id": int} - api: LibreNMSAPI instance for device fetching - sync_options: Optional dict with use_sysname, strip_domain settings - libre_devices_cache: Optional pre-fetched device data cache - job: Optional JobRunner instance for background job logging/cancellation - user: User performing the import (for permission checks). If job is provided, - user is extracted from job.job.user if not explicitly passed. - - Returns: - Dict with keys: - - success: List of {"device_id": int, "device": VM, "message": str} - - failed: List of {"device_id": int, "error": str} - - skipped: List of {"device_id": int, "reason": str} - - Raises: - PermissionDenied: If user lacks required permissions - - Example: - >>> # Synchronous import from view - >>> vm_imports = {123: {"cluster_id": 5, "device_role_id": 2}} - >>> result = bulk_import_vms(vm_imports, api, sync_options, user=request.user) - >>> print(f"Created {len(result['success'])} VMs") - >>> - >>> # Background job import - >>> result = bulk_import_vms(vm_imports, api, sync_options, cache, job=self) - """ - from netbox_librenms_plugin.import_validation_helpers import ( - apply_cluster_to_validation, - apply_role_to_validation, - ) - - # Extract user from job if not explicitly provided - if user is None and job is not None: - user = getattr(job.job, "user", None) - - # Check permissions at start of bulk operation - require_permissions(user, ["virtualization.add_virtualmachine"], "import VMs") - - result = {"success": [], "failed": [], "skipped": []} - vm_ids = list(vm_imports.keys()) - - # Use job logger if available, otherwise standard logger - log = job.logger if job else logger - - for idx, vm_id in enumerate(vm_ids, start=1): - # Check for job cancellation every 5 VMs - if job and idx % 5 == 0: - job.job.refresh_from_db() - job_status = job.job.status - status_value = job_status.value if hasattr(job_status, "value") else job_status - if status_value in ("failed", "errored"): - log.warning(f"Job cancelled at VM {idx} of {len(vm_ids)}") - break - log.info(f"Imported VM {idx} of {len(vm_ids)}") - - try: - # Fetch device data (uses cache helper) - libre_device = fetch_device_with_cache(vm_id, api, api.server_key, libre_devices_cache) - - if not libre_device: - result["failed"].append( - { - "device_id": vm_id, - "error": f"Device {vm_id} not found in LibreNMS", - } - ) - log.error(f"Device {vm_id} not found in LibreNMS") - continue - - # Validate as VM - validation = validate_device_for_import(libre_device, import_as_vm=True, api=api) - - # Check if VM already exists - if validation.get("existing_device"): - result["skipped"].append( - { - "device_id": vm_id, - "reason": f"VM already exists: {validation['existing_device'].name}", - } - ) - log.info(f"VM already exists: {validation['existing_device'].name}") - continue - - # Apply manual cluster and role selections - vm_mappings = vm_imports[vm_id] - cluster_id = vm_mappings.get("cluster_id") - role_id = vm_mappings.get("device_role_id") - - if cluster_id: - cluster = Cluster.objects.filter(id=cluster_id).first() - if cluster: - apply_cluster_to_validation(validation, cluster) - - role = None - if role_id: - role = DeviceRole.objects.filter(id=role_id).first() - if role: - apply_role_to_validation(validation, role, is_vm=True) - - # Determine VM name - use_sysname = sync_options.get("use_sysname", True) if sync_options else True - strip_domain = sync_options.get("strip_domain", False) if sync_options else False - - vm_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=strip_domain, - device_id=vm_id, - ) - - # Update validation with computed name - libre_device["_computed_name"] = vm_name - - # Create VM - vm = create_vm_from_librenms(libre_device, validation, use_sysname=use_sysname, role=role) - - result["success"].append( - { - "device_id": vm_id, - "device": vm, - "message": f"VM {vm.name} created successfully", - } - ) - log.info(f"Successfully imported VM {vm.name} (ID: {vm_id})") - - except Exception as vm_error: - log.error(f"Failed to import VM {vm_id}: {vm_error}", exc_info=True) - result["failed"].append({"device_id": vm_id, "error": str(vm_error)}) - - return result - - -def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> dict: - """ - Detect if device is a stack/Virtual Chassis by analyzing ENTITY-MIB inventory. - Vendor-agnostic using standard hierarchical structure. - - Args: - api: LibreNMSAPI instance - device_id: LibreNMS device ID - - Returns: - dict with structure: - { - 'is_stack': bool, - 'member_count': int, - 'members': [ - { - 'serial': str, - 'position': int, - 'model': str, - 'name': str, - 'index': int, - 'description': str, - 'suggested_name': str # Generated using master device name - } - ] - } - Returns None if not a stack or detection fails. - - Detection Logic: - 1. Check root level (entPhysicalContainedIn=0) for parent container - 2. Find parent index (entPhysicalClass='stack' or 'chassis') - 3. Get children chassis at that parent's index - 4. If multiple chassis found β†’ Stack detected - """ - try: - # Get the master device info to use for naming - success, device_info = api.get_device_info(device_id) - master_name = None - if success and device_info: - master_name = device_info.get("sysName") or device_info.get("hostname") - - # Step 1: Get root level items - success, root_items = api.get_inventory_filtered(device_id, ent_physical_contained_in=0) - - if not success or not root_items: - logger.debug(f"No root inventory items found for device {device_id}") - return None - - # Step 2: Find parent container index - # Could be class="stack" or the main "chassis" - parent_index = None - for item in root_items: - item_class = item.get("entPhysicalClass") - if item_class in ["stack", "chassis"]: - parent_index = item.get("entPhysicalIndex") - logger.debug(f"VC detection: Found parent container at index {parent_index} for device {device_id}") - break - - if not parent_index: - return None - - # Step 3: Get children chassis at next level - success, child_items = api.get_inventory_filtered( - device_id, - ent_physical_class="chassis", - ent_physical_contained_in=parent_index, - ) - - if not success: - return None - - # Filter for chassis only (in case API filter didn't work) - chassis_items = [item for item in (child_items or []) if item.get("entPhysicalClass") == "chassis"] - - # Step 4: Multiple chassis = stack - if len(chassis_items) <= 1: - return None - - # Step 5: Extract member info - members = [] - for idx, chassis in enumerate(chassis_items): - raw_position = chassis.get("entPhysicalParentRelPos", idx) - try: - position = int(raw_position) - except (TypeError, ValueError): - position = idx - member_data = { - "serial": chassis.get("entPhysicalSerialNum", ""), - "position": position, - "model": chassis.get("entPhysicalModelName", ""), - "name": chassis.get("entPhysicalName", ""), - "index": chassis.get("entPhysicalIndex"), - "description": chassis.get("entPhysicalDescr", ""), - } - - # Generate suggested name if we have master name - if master_name: - member_data["suggested_name"] = _generate_vc_member_name(master_name, position + 1) - else: - member_data["suggested_name"] = f"Member-{position + 1}" - - members.append(member_data) - - # Sort by position - members.sort(key=lambda m: m["position"]) - - logger.info(f"Detected stack with {len(members)} members for device {device_id}") - - return {"is_stack": True, "member_count": len(members), "members": members} - - except Exception as e: - logger.exception(f"Error detecting virtual chassis for device {device_id}: {e}") - return None - - -def _generate_vc_member_name(master_name: str, position: int, serial: str = None) -> str: - """ - Generate name for VC member device using configured pattern from settings. - - Args: - master_name: Name of the master/primary device - position: VC position number - serial: Optional serial number of the member device - - Returns: - Generated member device name - - Examples: - pattern="-M{position}" β†’ "switch01-M2" - pattern=" ({position})" β†’ "switch01 (2)" - pattern="-SW{position}" β†’ "switch01-SW2" - pattern=" [{serial}]" β†’ "switch01 [ABC123]" - """ - # Import here to avoid circular dependency - from .models import LibreNMSSettings - - # Get pattern from settings with fallback to default - try: - settings = LibreNMSSettings.objects.first() - pattern = settings.vc_member_name_pattern if settings else "-M{position}" - except Exception as e: - logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") - pattern = "-M{position}" - - # Prepare format variables - format_vars = { - "master_name": master_name, - "position": position, - "serial": serial or "", - } - - # Apply pattern - pattern should be suffix/prefix, not full name - try: - formatted_suffix = pattern.format(**format_vars) - return f"{master_name}{formatted_suffix}" - except KeyError as e: - logger.error(f"Invalid placeholder in VC naming pattern '{pattern}': {e}. Using default.") - return f"{master_name}-M{position}" - - -def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: - """ - Regenerate suggested VC member names using the actual master device name. - - This ensures preview shows accurate names after use_sysname and strip_domain - are applied to the master device name. - - Args: - vc_data: Virtual chassis detection data dict - master_name: The actual name that will be used for master device in NetBox - - Returns: - Updated vc_data dict with corrected suggested_name for each member - """ - if not vc_data or not vc_data.get("is_stack"): - return vc_data - - for idx, member in enumerate(vc_data.get("members", [])): - raw_position = member.get("position", idx) - try: - base_position = int(raw_position) - except (TypeError, ValueError): - base_position = idx - position = base_position + 1 # Convert to 1-based position - member["position"] = base_position - member["suggested_name"] = _generate_vc_member_name(master_name, position, serial=member.get("serial")) - - return vc_data - - -def create_virtual_chassis_with_members(master_device: Device, members_info: list, libre_device: dict): - """ - Create Virtual Chassis and member devices from detection info. - - This function creates a NetBox VirtualChassis with the master device - and all detected member devices, wrapped in a transaction for safety. - - Args: - master_device: The imported device (becomes VC master) - members_info: List of member dicts from VC detection - libre_device: Original LibreNMS device data - - Returns: - VirtualChassis: The created virtual chassis instance - - Raises: - ValidationError: If member count validation fails - IntegrityError: If duplicate serials/names are detected - Exception: For other creation errors - - Example members_info: - [ - {'serial': 'ABC123', 'position': 0, 'model': 'C9300-48U', 'name': 'Switch 1'}, - {'serial': 'ABC124', 'position': 1, 'model': 'C9300-48U', 'name': 'Switch 2'} - ] - """ - - # Store original master device state for rollback - original_master_name = master_device.name - original_vc = master_device.virtual_chassis - original_vc_position = master_device.vc_position - - try: - with transaction.atomic(): - # Rename master device to include position 1 pattern - master_device_new_name = _generate_vc_member_name(original_master_name, 1, serial=master_device.serial) - - # Check if renamed master conflicts with existing device - if Device.objects.filter(name=master_device_new_name).exclude(pk=master_device.pk).exists(): - logger.warning( - f"Cannot rename master to '{master_device_new_name}' - name already exists. " - f"Keeping original name '{original_master_name}'" - ) - master_base_name = original_master_name - else: - master_device.name = master_device_new_name - master_base_name = original_master_name - - # Create VC using original base name - vc_name = master_base_name - vc = VirtualChassis.objects.create( - name=vc_name, - master=master_device, - domain=f"librenms-{libre_device['device_id']}", - ) - - # Update master device - master_device.virtual_chassis = vc - master_device.vc_position = 1 # Master is position 1 - master_device.save() - - # Create member devices for remaining positions - position = 2 # Start at 2 (master is 1) - members_created = 0 - - for member in members_info: - # Skip if this is the master's serial - if member.get("serial") == master_device.serial: - continue - - serial = member.get("serial") - - member_rack = master_device.rack - member_location = master_device.location or ( - member_rack.location if member_rack and member_rack.location else None - ) - - # Check for duplicate serial - if serial and Device.objects.filter(serial=serial).exists(): - logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") - continue - - member_name = _generate_vc_member_name(master_base_name, position, serial=serial) - - # Check for duplicate name - if Device.objects.filter(name=member_name).exists(): - logger.warning(f"Device with name '{member_name}' already exists, skipping VC member creation") - continue - - Device.objects.create( - name=member_name, - device_type=master_device.device_type, - role=master_device.role, - site=master_device.site, - location=member_location, - rack=member_rack, - platform=master_device.platform, - serial=serial, - virtual_chassis=vc, - vc_position=position, - comments=f"VC member (LibreNMS: {member.get('name', 'Unknown')})\n" - f"Auto-created from stack inventory", - ) - members_created += 1 - position += 1 - - # Validate member count - expected_members = len([m for m in members_info if m.get("serial") != master_device.serial]) - if members_created < expected_members: - logger.warning( - f"Created {members_created} members but expected {expected_members}. " - "Some members may have been skipped due to duplicates." - ) - - logger.info( - f"Created Virtual Chassis '{vc.name}' with {vc.members.count()} total members " - f"(1 master + {members_created} additional)" - ) - - return vc - - except Exception as e: - # Rollback master device to original state - logger.error( - f"Virtual Chassis creation failed for device {master_device.name}: {e}. Rolling back master device changes." - ) - master_device.name = original_master_name - master_device.virtual_chassis = original_vc - master_device.vc_position = original_vc_position - master_device.save() - raise - - -def process_device_filters( - api: LibreNMSAPI, - filters: dict, - vc_detection_enabled: bool, - clear_cache: bool, - show_disabled: bool, - exclude_existing: bool = False, - job=None, - request=None, - return_cache_status: bool = False, -) -> List[dict] | tuple[List[dict], bool]: - """ - Process LibreNMS device filters and return validated devices. - - Shared function used by both synchronous view and background job processing. - Fetches devices, optionally pre-warms VC cache, validates each device, and - caches results for HTMX row updates. - - Args: - api: LibreNMS API client instance - filters: Filter dict with location, type, os, hostname, sysname, hardware keys - vc_detection_enabled: Whether to detect virtual chassis - clear_cache: Whether to force cache refresh - show_disabled: Whether to include disabled devices - exclude_existing: Whether to exclude devices that already exist in NetBox - job: Optional JobRunner instance for logging job events - request: Optional Django request for client disconnect detection (synchronous only) - return_cache_status: When True, returns (devices, from_cache) tuple - - Returns: - List[dict]: Validated devices with _validation key, or tuple of (devices, from_cache) - if return_cache_status is True. from_cache=True means data was loaded from existing - cache; from_cache=False means data was just fetched from LibreNMS. - """ - # Fetch devices from LibreNMS - if job: - job.logger.info(f"Fetching devices with filters: {filters}") - else: - logger.info(f"Fetching devices with filters: {filters}") - - # Always get cache status internally, even if not returning it - # We need it to determine if metadata should be updated - libre_devices, from_cache = get_librenms_devices_for_import( - api, - filters=filters, - force_refresh=clear_cache, - return_cache_status=True, - ) - - # Filter out disabled devices if requested - if not show_disabled: - libre_devices = [d for d in libre_devices if d.get("status") == 1] - - if job: - job.logger.info(f"Found {len(libre_devices)} devices to process") - else: - logger.info(f"Found {len(libre_devices)} devices") - - # Pre-warm VC cache if needed - if vc_detection_enabled and libre_devices: - device_ids = [d["device_id"] for d in libre_devices] - if job: - job.logger.info( - f"Pre-fetching virtual chassis data for {len(device_ids)} devices. This may take some time..." - ) - else: - logger.info(f"Pre-fetching VC data for {len(device_ids)} devices") - - try: - prefetch_vc_data_for_devices(api, device_ids, force_refresh=clear_cache) - if job: - job.logger.info("Virtual chassis data pre-fetch completed") - except (BrokenPipeError, ConnectionError, IOError) as e: - if request: - logger.info(f"Client disconnected during VC prefetch: {e}") - return [] - raise - - # Validate each device - validated_devices = [] - total = len(libre_devices) - api_for_validation = api if vc_detection_enabled else None - - if job: - job.logger.info(f"Starting validation of {total} devices") - # Initial check if job was already terminated before we even started - try: - from django_rq import get_queue - from rq.job import Job as RQJob - - queue = get_queue("default") - rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) - - if rq_job.is_failed or rq_job.is_stopped: - job.logger.warning("Job was already stopped before validation started") - return [] - except Exception: - # Fall back to DB check if RQ check fails - job.job.refresh_from_db() - if job.job.status == JobStatusChoices.STATUS_FAILED: - job.logger.warning("Job was stopped before validation started") - return [] - else: - logger.info(f"Validating {total} devices") - - for idx, device in enumerate(libre_devices, 1): - # Check for job termination or client disconnect periodically - if idx % 5 == 0 or idx == 1: # Check more frequently (every 5 devices + first device) - if job: - # Check if job was terminated via stop API - # CRITICAL: Check the RQ job status in Redis, not just the DB model - # NetBox's stop endpoint marks the RQ job as failed in Redis - try: - from django_rq import get_queue - from rq.job import Job as RQJob - - queue = get_queue("default") - rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) - - # Check if RQ job is in a stopped state - if rq_job.is_failed or rq_job.is_stopped: - job.logger.info( - f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." - ) - return [] - except Exception: - # If we can't check RQ status, fall back to DB status check - job.job.refresh_from_db() - if job.job.status == JobStatusChoices.STATUS_FAILED: - job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") - return [] - elif request: - # Check for client disconnect - try: - if hasattr(request, "META") and request.META.get("wsgi.input"): - pass - except (BrokenPipeError, ConnectionError, IOError): - logger.info(f"Client disconnected during validation at device {idx}") - return [] - - # Drop any cached validation/meta keys before recomputing - device.pop("_validation", None) - - # Generate shared cache key for this validated device - device_id = device["device_id"] - cache_key = get_validated_device_cache_key( - server_key=api.server_key, - filters=filters, - device_id=device_id, - vc_enabled=vc_detection_enabled, - ) - - # Check if we already have cached validation for this device - # (only if not forcing refresh) - if not clear_cache: - cached_device = cache.get(cache_key) - if cached_device: - # Use cached validation - device["_validation"] = cached_device["_validation"] - - # Apply exclude_existing filter if enabled - if exclude_existing: - validation = device["_validation"] - if validation["existing_device"]: - continue - - validated_devices.append(device) - continue - - # Not in cache or forcing refresh - validate now - try: - validation = validate_device_for_import( - device, - api=api_for_validation, - include_vc_detection=vc_detection_enabled, - force_vc_refresh=clear_cache, - ) - except (BrokenPipeError, ConnectionError, IOError) as e: - if request: - logger.info(f"Client disconnected during device validation: {e}") - return [] - raise - - # Set VC detection metadata - if not vc_detection_enabled: - validation["virtual_chassis"] = empty_virtual_chassis_data() - - # Apply exclude_existing filter if enabled - if exclude_existing and validation["existing_device"]: - continue - - device["_validation"] = validation - validated_devices.append(device) - - # Cache with TWO keys for different purposes: - # 1. Complex key (with filter context) - for full validated device with all metadata - cache.set(cache_key, device, timeout=api.cache_timeout) - - # 2. Simple key (device ID only) - for quick device data lookup by role/rack updates - # This avoids redundant API calls when user interacts with dropdowns - simple_cache_key = get_import_device_cache_key(device_id, api.server_key) - # Cache just the raw device data (not the full validation result) - # This is what get_validated_device_with_selections() expects - device_data_only = {k: v for k, v in device.items() if k != "_validation"} - cache.set(simple_cache_key, device_data_only, timeout=api.cache_timeout) - - # Store cache metadata (timestamp) for all filter operations - # This enables countdown display regardless of background job vs synchronous execution - # Always store metadata when we have validated devices, even if from_cache - # This ensures metadata is available for countdown display - if validated_devices: - from datetime import datetime, timezone - - cache_metadata_key = get_cache_metadata_key( - server_key=api.server_key, filters=filters, vc_enabled=vc_detection_enabled - ) - - # Check if metadata already exists to preserve original timestamp - # BUT: if clear_cache was requested or data came fresh from LibreNMS, update it - existing_metadata = cache.get(cache_metadata_key) - should_update = clear_cache or not from_cache - - if existing_metadata and not should_update: - # Metadata exists and cache wasn't cleared, keep using it (preserves original cache time) - pass - else: - # No metadata exists, OR cache was cleared, OR fresh data - create/update it now - cache_metadata = { - "cached_at": datetime.now(timezone.utc).isoformat(), - "cache_timeout": api.cache_timeout, - "filters": filters, - "vc_enabled": vc_detection_enabled, - "device_count": len(validated_devices), - } - cache.set(cache_metadata_key, cache_metadata, timeout=api.cache_timeout) - - # Maintain cache index for this server to enable listing active searches - cache_index_key = f"librenms_cache_index_{api.server_key}" - cache_index = cache.get(cache_index_key, []) - # Add this cache key if not already in index - if cache_metadata_key not in cache_index: - cache_index.append(cache_metadata_key) - # Store index with same timeout as the metadata - cache.set(cache_index_key, cache_index, timeout=api.cache_timeout) - - if job: - if exclude_existing: - filtered_count = total - len(validated_devices) - job.logger.info( - f"Validation complete: {len(validated_devices)} devices passed filter, " - f"{filtered_count} filtered out (existing devices excluded)" - ) - else: - job.logger.info(f"Validation complete: {len(validated_devices)} devices ready for import") - else: - logger.info(f"Processed {len(validated_devices)} validated devices") - - if return_cache_status: - return validated_devices, from_cache - return validated_devices diff --git a/netbox_librenms_plugin/import_utils/__init__.py b/netbox_librenms_plugin/import_utils/__init__.py new file mode 100644 index 0000000000..b7a08fef59 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/__init__.py @@ -0,0 +1,52 @@ +""" +Utilities for importing devices from LibreNMS to NetBox. + +This package provides functions for: +- Validating LibreNMS devices for import +- Retrieving filtered LibreNMS devices +- Importing single and multiple devices +- Smart matching of NetBox objects +- Permission checking for import operations +- Virtual chassis detection and creation + +All imports below are intentional re-exports so that existing callers +can continue using ``from netbox_librenms_plugin.import_utils import X``. +The F401 suppressions prevent linters from flagging them as unused. +""" + +from .bulk_import import ( # noqa: F401 + bulk_import_devices, + bulk_import_devices_shared, + process_device_filters, +) +from .cache import ( # noqa: F401 + get_active_cached_searches, + get_cache_metadata_key, + get_import_device_cache_key, + get_validated_device_cache_key, +) +from .device_operations import ( # noqa: F401 + _determine_device_name, + fetch_device_with_cache, + get_librenms_device_by_id, + import_single_device, + validate_device_for_import, +) +from .filters import ( # noqa: F401 + _apply_client_filters, + get_device_count_for_filters, + get_librenms_devices_for_import, +) +from .permissions import check_user_permissions, require_permissions # noqa: F401 +from .virtual_chassis import ( # noqa: F401 + _clone_virtual_chassis_data, + _generate_vc_member_name, + _vc_cache_key, + create_virtual_chassis_with_members, + detect_virtual_chassis_from_inventory, + empty_virtual_chassis_data, + get_virtual_chassis_data, + prefetch_vc_data_for_devices, + update_vc_member_suggested_names, +) +from .vm_operations import bulk_import_vms, create_vm_from_librenms # noqa: F401 diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py new file mode 100644 index 0000000000..5072070c56 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -0,0 +1,661 @@ +"""Bulk import orchestration for devices and filter processing.""" + +import logging +from typing import List + +from core.choices import JobStatusChoices +from django.core.cache import cache + +from ..librenms_api import LibreNMSAPI +from ..utils import find_by_librenms_id +from .cache import get_cache_metadata_key, get_import_device_cache_key, get_validated_device_cache_key +from .device_operations import import_single_device, validate_device_for_import +from .filters import get_librenms_devices_for_import +from .permissions import require_permissions +from .virtual_chassis import ( + create_virtual_chassis_with_members, + empty_virtual_chassis_data, + prefetch_vc_data_for_devices, +) + +logger = logging.getLogger(__name__) + + +def _empty_return(return_cache_status: bool): + """Centralised empty-result return value for process_device_filters.""" + return ([], False) if return_cache_status else [] + + +def bulk_import_devices_shared( + device_ids: List[int], + server_key: str = None, + sync_options: dict = None, + manual_mappings_per_device: dict = None, + libre_devices_cache: dict = None, + job=None, + user=None, +) -> dict: + """ + Shared function for importing multiple LibreNMS devices to NetBox. + + Used by both synchronous imports and background jobs. Handles per-device error + collection and optional progress logging when job context is provided. + + Args: + device_ids: List of LibreNMS device IDs to import + server_key: LibreNMS server configuration key + sync_options: Sync options to apply to all devices + manual_mappings_per_device: Dict mapping device_id to manual_mappings dict + Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} + libre_devices_cache: Optional dict mapping device_id to pre-fetched device data + to avoid redundant API calls. Example: {123: {...device_data...}} + job: Optional JobRunner instance for progress logging and cancellation checks + user: User performing the import (for permission checks). If job is provided, + user is extracted from job.job.user if not explicitly passed. + + Returns: + dict: Bulk import result with structure: + { + 'total': int, + 'success': List[dict], # Successfully imported devices + 'failed': List[dict], # Failed imports with errors + 'skipped': List[dict], # Skipped devices (already exist, etc.) + 'virtual_chassis_created': int # Number of VCs created + } + + Raises: + PermissionDenied: If user lacks required permissions + + Example: + >>> # Synchronous usage + >>> result = bulk_import_devices_shared([1, 2, 3, 4, 5], user=request.user) + >>> # Background job usage + >>> result = bulk_import_devices_shared([1, 2, 3], job=self) + """ + # Extract user from job if not explicitly provided + if user is None and job is not None: + user = getattr(job.job, "user", None) + + # Check permissions at start of bulk operation + required_perms = [ + "dcim.add_device", + "dcim.add_interface", + "dcim.add_virtualchassis", + ] + require_permissions(user, required_perms, "import devices") + + total = len(device_ids) + success_list = [] + failed_list = [] + skipped_list = [] + vc_created_count = 0 + processed_vc_domains = set() # Track VCs already created by domain + + # Initialize API client once for all devices to avoid repeated config parsing + api = LibreNMSAPI(server_key=server_key) + + for idx, device_id in enumerate(device_ids, start=1): + # Check for job cancellation on first iteration and every 5th thereafter. + # Check RQ/Redis state first (reflects stop API immediately); fall back to DB. + if job and (idx == 1 or idx % 5 == 0): + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + if rq_job.is_failed or rq_job.is_stopped: + if job.logger: + job.logger.warning( + f"Import job stopped at device {idx} of {total} (RQ status: {rq_job.get_status()})" + ) + else: + logger.warning(f"Import cancelled at device {idx} of {total}") + break + except Exception: + # Fall back to DB check if RQ is unavailable + job.job.refresh_from_db() + job_status = job.job.status + status_value = job_status.value if hasattr(job_status, "value") else job_status + if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): + if job.logger: + job.logger.warning(f"Import job cancelled at device {idx} of {total}") + else: + logger.warning(f"Import cancelled at device {idx} of {total}") + break + + try: + # Use cached device data if available to avoid redundant API calls + if libre_devices_cache and device_id in libre_devices_cache: + libre_device = libre_devices_cache[device_id] + success = True + else: + success, libre_device = api.get_device_info(device_id) + + if not success or not libre_device: + error_msg = f"Failed to retrieve device {device_id} from LibreNMS" + failed_list.append({"device_id": device_id, "error": error_msg}) + if job and job.logger: + job.logger.error(error_msg) + else: + logger.error(error_msg) + continue + + use_sysname_opt = sync_options.get("use_sysname", True) if sync_options else True + strip_domain_opt = sync_options.get("strip_domain", False) if sync_options else False + validation = validate_device_for_import( + libre_device, + api=api, + use_sysname=use_sysname_opt, + strip_domain=strip_domain_opt, + server_key=api.server_key, + ) + + # Build manual mappings from validation + any provided overrides + device_mappings = {} + + # Get site and device_type from validation + if validation["site"].get("found") and validation["site"].get("site"): + device_mappings["site_id"] = validation["site"]["site"].id + if validation["device_type"].get("found") and validation["device_type"].get("device_type"): + device_mappings["device_type_id"] = validation["device_type"]["device_type"].id + if validation["platform"].get("found") and validation["platform"].get("platform"): + device_mappings["platform_id"] = validation["platform"]["platform"].id + + # Override with any manual mappings provided for this device + if manual_mappings_per_device and device_id in manual_mappings_per_device: + device_mappings.update(manual_mappings_per_device[device_id]) + + result = import_single_device( + device_id, + server_key=api.server_key, # use resolved key, not raw parameter (may be None) + sync_options=sync_options, + manual_mappings=device_mappings if device_mappings else None, + libre_device=libre_device, + ) + + if result["success"]: + success_list.append( + { + "device_id": device_id, + "device": result["device"], + "message": result["message"], + } + ) + # Log progress after each successful import + if job and job.logger: + job.logger.info(f"Imported device {idx} of {total}") + + # Handle virtual chassis creation for stacks + vc_data = validation.get("virtual_chassis", {}) + if vc_data.get("is_stack", False): + # Derive a stack-level dedup key from member serials so that all + # LibreNMS devices belonging to the same physical stack (e.g. each + # switch in a stacked chassis that appears as a separate device in + # LibreNMS) share the same key and VC creation is triggered only once. + # Fall back to device_id when no member serials are available. + member_serials = sorted(m.get("serial") for m in vc_data.get("members", []) if m.get("serial")) + vc_domain = ( + f"librenms-stack-{','.join(member_serials)}" if member_serials else f"librenms-{device_id}" + ) + + # Only create VC if we haven't processed this stack yet + # Add to set BEFORE attempting creation to prevent race condition + if vc_domain not in processed_vc_domains: + processed_vc_domains.add(vc_domain) + try: + vc = create_virtual_chassis_with_members( + result["device"], + vc_data["members"], + libre_device, + ) + vc_created_count += 1 + log_msg = f"Created VC '{vc.name}' during bulk import for device {device_id}" + if job and job.logger: + job.logger.info(log_msg) + else: + logger.info(log_msg) + except Exception as vc_error: + # Remove from set on failure so retry is possible + processed_vc_domains.discard(vc_domain) + warn_msg = f"Failed to create VC for device {device_id}: {vc_error}" + if job and job.logger: + job.logger.warning(warn_msg) + else: + logger.warning(warn_msg) + # Don't fail the import, just log the warning + + elif result.get("device"): # Device exists + skipped_list.append({"device_id": device_id, "reason": result["error"]}) + else: # Failed to import + failed_list.append({"device_id": device_id, "error": result["error"]}) + if job and job.logger: + job.logger.error(f"Failed to import device {device_id}: {result['error']}") + + except Exception as e: + error_msg = f"Unexpected error importing device {device_id}: {str(e)}" + if job and job.logger: + job.logger.error(error_msg, exc_info=True) + else: + logger.exception(f"Unexpected error importing device {device_id}") + failed_list.append({"device_id": device_id, "error": str(e)}) + + return { + "total": total, + "success": success_list, + "failed": failed_list, + "skipped": skipped_list, + "virtual_chassis_created": vc_created_count, + } + + +def bulk_import_devices( + device_ids: List[int], + server_key: str = None, + sync_options: dict = None, + manual_mappings_per_device: dict = None, + libre_devices_cache: dict = None, + user=None, +) -> dict: + """ + Import multiple LibreNMS devices to NetBox (synchronous). + + This is the public API for synchronous imports. For background job usage, + use bulk_import_devices_shared() with a job context. + + Args: + device_ids: List of LibreNMS device IDs to import + server_key: LibreNMS server configuration key + sync_options: Sync options to apply to all devices + manual_mappings_per_device: Dict mapping device_id to manual_mappings dict + Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} + libre_devices_cache: Optional dict mapping device_id to pre-fetched device data + to avoid redundant API calls. Example: {123: {...device_data...}} + user: User performing the import (for permission checks) + + Returns: + dict: Bulk import result with structure: + { + 'total': int, + 'success': List[dict], # Successfully imported devices + 'failed': List[dict], # Failed imports with errors + 'skipped': List[dict], # Skipped devices (already exist, etc.) + 'virtual_chassis_created': int # Number of VCs created + } + + Raises: + PermissionDenied: If user lacks required permissions + """ + return bulk_import_devices_shared( + device_ids=device_ids, + server_key=server_key, + sync_options=sync_options, + manual_mappings_per_device=manual_mappings_per_device, + libre_devices_cache=libre_devices_cache, + job=None, # No job context for synchronous imports + user=user, + ) + + +def _refresh_existing_device(validation: dict, libre_device: dict = None, server_key: str = "default") -> None: + """Refresh existing_device from DB to pick up changes made in NetBox since caching. + + When existing_device is None (wasn't found at cache time), re-check if the device + was imported since caching by looking up librenms_id or hostname. + """ + existing = validation.get("existing_device") + if existing and hasattr(existing, "pk"): + try: + from dcim.models import Device + from virtualization.models import VirtualMachine + + if validation.get("import_as_vm"): + refreshed = VirtualMachine.objects.filter(pk=existing.pk).first() + else: + refreshed = Device.objects.filter(pk=existing.pk).first() + + if refreshed: + validation["existing_device"] = refreshed + if hasattr(refreshed, "role") and refreshed.role: + validation["device_role"] = {"found": True, "role": refreshed.role} + else: + # Device was deleted since caching β€” recompute readiness to match + # validate_device_for_import logic. + validation["existing_device"] = None + validation["existing_match_type"] = None + can_import = not bool(validation.get("issues")) + if validation.get("import_as_vm"): + # VMs only require a cluster (site/role not mandatory) + is_ready = can_import and bool(validation.get("cluster", {}).get("found")) + else: + is_ready = ( + can_import + and bool(validation.get("site", {}).get("found")) + and bool(validation.get("device_type", {}).get("found")) + and bool(validation.get("device_role", {}).get("found")) + ) + validation["can_import"] = can_import + validation["is_ready"] = is_ready + except Exception as e: + existing_id = getattr(existing, "pk", "unknown") if existing else "none" + logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") + return + + # existing_device was None at cache time β€” check if device was imported since + if not libre_device: + return + try: + from dcim.models import Device + from virtualization.models import VirtualMachine + + import_as_vm = validation.get("import_as_vm", False) + Model = VirtualMachine if import_as_vm else Device + + librenms_id = libre_device.get("device_id") + hostname = libre_device.get("hostname", "") + sys_name = libre_device.get("sysName", "") + + new_device = None + match_type = None + + # Check by librenms_id custom field first (JSON multi-server format + legacy) + if librenms_id: + try: + new_device = find_by_librenms_id(Model, int(librenms_id), server_key) + if new_device: + match_type = "librenms_id" + except (ValueError, TypeError): + pass + + # Fall back to hostname match, then sys_name independently + if not new_device and hostname: + new_device = Model.objects.filter(name__iexact=hostname).first() + if new_device: + match_type = "hostname" + if not new_device and sys_name: + new_device = Model.objects.filter(name__iexact=sys_name).first() + if new_device: + match_type = "hostname" + + if new_device: + validation["existing_device"] = new_device + validation["existing_match_type"] = match_type + validation["can_import"] = False + validation["is_ready"] = False + if not import_as_vm and hasattr(new_device, "role") and new_device.role: + validation["device_role"] = {"found": True, "role": new_device.role} + except Exception as e: + logger.error(f"Failed to check for newly imported device: {e}") + + +def process_device_filters( + api: LibreNMSAPI, + filters: dict, + vc_detection_enabled: bool, + clear_cache: bool, + show_disabled: bool, + exclude_existing: bool = False, + job=None, + request=None, + return_cache_status: bool = False, + use_sysname: bool = True, + strip_domain: bool = False, +) -> List[dict] | tuple[List[dict], bool]: + """ + Process LibreNMS device filters and return validated devices. + + Shared function used by both synchronous view and background job processing. + Fetches devices, optionally pre-warms VC cache, validates each device, and + caches results for HTMX row updates. + + Args: + api: LibreNMS API client instance + filters: Filter dict with location, type, os, hostname, sysname, hardware keys + vc_detection_enabled: Whether to detect virtual chassis + clear_cache: Whether to force cache refresh + show_disabled: Whether to include disabled devices + exclude_existing: Whether to exclude devices that already exist in NetBox + job: Optional JobRunner instance for logging job events + request: Optional Django request for client disconnect detection (synchronous only) + return_cache_status: When True, returns (devices, from_cache) tuple + use_sysname: If True, prefer sysName over hostname for device name resolution + strip_domain: If True, strip domain suffix from device name + + Returns: + List[dict]: Validated devices with _validation key, or tuple of (devices, from_cache) + if return_cache_status is True. from_cache=True means data was loaded from existing + cache; from_cache=False means data was just fetched from LibreNMS. + """ + # Fetch devices from LibreNMS + if job: + job.logger.info(f"Fetching devices with filters: {filters}") + else: + logger.info(f"Fetching devices with filters: {filters}") + + # Always get cache status internally, even if not returning it + # We need it to determine if metadata should be updated + libre_devices, from_cache = get_librenms_devices_for_import( + api, + filters=filters, + force_refresh=clear_cache, + return_cache_status=True, + ) + + # Filter out disabled devices if requested. LibreNMS's "disabled" field (1=disabled, + # 0=enabled) reflects manual device disablement; "status" reflects SNMP reachability. + # show_disabled controls the former: hidden when disabled==1, shown regardless of status. + if not show_disabled: + libre_devices = [d for d in libre_devices if int(d.get("disabled", 0)) != 1] + + if job: + job.logger.info(f"Found {len(libre_devices)} devices to process") + else: + logger.info(f"Found {len(libre_devices)} devices") + + # Pre-warm VC cache if needed + if vc_detection_enabled and libre_devices: + device_ids = [d["device_id"] for d in libre_devices] + if job: + job.logger.info( + f"Pre-fetching virtual chassis data for {len(device_ids)} devices. This may take some time..." + ) + else: + logger.info(f"Pre-fetching VC data for {len(device_ids)} devices") + + try: + prefetch_vc_data_for_devices(api, device_ids, force_refresh=clear_cache) + if job: + job.logger.info("Virtual chassis data pre-fetch completed") + except (BrokenPipeError, ConnectionError, IOError) as e: + if request: + logger.info(f"Client disconnected during VC prefetch: {e}") + return _empty_return(return_cache_status) + raise + + # Validate each device + validated_devices = [] + total = len(libre_devices) + api_for_validation = api if vc_detection_enabled else None + + if job: + job.logger.info(f"Starting validation of {total} devices") + # Initial check if job was already terminated before we even started + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + + if rq_job.is_failed or rq_job.is_stopped: + job.logger.warning("Job was already stopped before validation started") + return _empty_return(return_cache_status) + except Exception: + # Fall back to DB check if RQ check fails + job.job.refresh_from_db() + if job.job.status in (JobStatusChoices.STATUS_FAILED, JobStatusChoices.STATUS_ERRORED): + job.logger.warning("Job was stopped before validation started") + return _empty_return(return_cache_status) + else: + logger.info(f"Validating {total} devices") + + for idx, device in enumerate(libre_devices, 1): + # Check for job termination or client disconnect periodically + if idx % 5 == 0 or idx == 1: # Check more frequently (every 5 devices + first device) + if job: + # Check if job was terminated via stop API + # CRITICAL: Check the RQ job status in Redis, not just the DB model + # NetBox's stop endpoint marks the RQ job as failed in Redis + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + + # Check if RQ job is in a stopped state + if rq_job.is_failed or rq_job.is_stopped: + job.logger.info( + f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." + ) + return _empty_return(return_cache_status) + except Exception: + # If we can't check RQ status, fall back to DB status check + job.job.refresh_from_db() + if job.job.status in (JobStatusChoices.STATUS_FAILED, JobStatusChoices.STATUS_ERRORED): + job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") + return _empty_return(return_cache_status) + + # Drop any cached validation/meta keys before recomputing + device.pop("_validation", None) + + # Generate shared cache key for this validated device + device_id = device["device_id"] + cache_key = get_validated_device_cache_key( + server_key=api.server_key, + filters=filters, + device_id=device_id, + vc_enabled=vc_detection_enabled, + use_sysname=use_sysname, + strip_domain=strip_domain, + ) + + # Check if we already have cached validation for this device + # (only if not forcing refresh) + if not clear_cache: + cached_device = cache.get(cache_key) + if cached_device: + # Use cached validation + device["_validation"] = cached_device["_validation"] + + # Refresh existing_device from DB to avoid stale data + # (user may have changed role, name, etc. in NetBox) + _refresh_existing_device(device["_validation"], libre_device=device, server_key=api.server_key) + + # Apply exclude_existing filter if enabled + if exclude_existing: + validation = device["_validation"] + if validation["existing_device"]: + continue + + validated_devices.append(device) + continue + + # Not in cache or forcing refresh - validate now + try: + validation = validate_device_for_import( + device, + api=api_for_validation, + include_vc_detection=vc_detection_enabled, + force_vc_refresh=clear_cache, + server_key=api.server_key, + use_sysname=use_sysname, + strip_domain=strip_domain, + ) + except (BrokenPipeError, ConnectionError, IOError) as e: + if request: + logger.info(f"Client disconnected during device validation: {e}") + return _empty_return(return_cache_status) + raise + + # Set VC detection metadata + if not vc_detection_enabled: + validation["virtual_chassis"] = empty_virtual_chassis_data() + + # Apply exclude_existing filter if enabled + if exclude_existing and validation["existing_device"]: + continue + + device["_validation"] = validation + validated_devices.append(device) + + # Cache with TWO keys for different purposes: + # 1. Complex key (with filter context) - for full validated device with all metadata + cache.set(cache_key, device, timeout=api.cache_timeout) + + # 2. Simple key (device ID only) - for quick device data lookup by role/rack updates + # This avoids redundant API calls when user interacts with dropdowns + simple_cache_key = get_import_device_cache_key(device_id, api.server_key) + # Cache just the raw device data (not the full validation result) + # This is what get_validated_device_with_selections() expects + device_data_only = {k: v for k, v in device.items() if k != "_validation"} + cache.set(simple_cache_key, device_data_only, timeout=api.cache_timeout) + + # Store cache metadata (timestamp) for all filter operations + # This enables countdown display regardless of background job vs synchronous execution + # Always store metadata when we have validated devices, even if from_cache + # This ensures metadata is available for countdown display + if validated_devices: + from datetime import datetime, timezone + + cache_metadata_key = get_cache_metadata_key( + server_key=api.server_key, + filters=filters, + vc_enabled=vc_detection_enabled, + use_sysname=use_sysname, + strip_domain=strip_domain, + ) + + # Check if metadata already exists to preserve original timestamp + # BUT: if clear_cache was requested or data came fresh from LibreNMS, update it + existing_metadata = cache.get(cache_metadata_key) + should_update = clear_cache or not from_cache + + if existing_metadata and not should_update: + # Metadata exists and cache wasn't cleared, keep using it (preserves original cache time) + pass + else: + # No metadata exists, OR cache was cleared, OR fresh data - create/update it now + cache_metadata = { + "cached_at": datetime.now(timezone.utc).isoformat(), + "cache_timeout": api.cache_timeout, + "filters": filters, + "vc_enabled": vc_detection_enabled, + "device_count": len(validated_devices), + } + cache.set(cache_metadata_key, cache_metadata, timeout=api.cache_timeout) + + # Maintain cache index for this server to enable listing active searches + cache_index_key = f"librenms_cache_index_{api.server_key}" + cache_index = cache.get(cache_index_key, []) + # Add this cache key if not already in index + if cache_metadata_key not in cache_index: + cache_index.append(cache_metadata_key) + # Store index with same timeout as the metadata + cache.set(cache_index_key, cache_index, timeout=api.cache_timeout) + + if job: + if exclude_existing: + filtered_count = total - len(validated_devices) + job.logger.info( + f"Validation complete: {len(validated_devices)} devices passed filter, " + f"{filtered_count} filtered out (existing devices excluded)" + ) + else: + job.logger.info(f"Validation complete: {len(validated_devices)} devices ready for import") + else: + logger.info(f"Processed {len(validated_devices)} validated devices") + + if return_cache_status: + return validated_devices, from_cache + return validated_devices diff --git a/netbox_librenms_plugin/import_utils/cache.py b/netbox_librenms_plugin/import_utils/cache.py new file mode 100644 index 0000000000..140f45f79d --- /dev/null +++ b/netbox_librenms_plugin/import_utils/cache.py @@ -0,0 +1,186 @@ +"""Cache key generation and management for device import operations.""" + +import hashlib +import json +import logging + +from django.core.cache import cache + +logger = logging.getLogger(__name__) + + +def get_cache_metadata_key( + server_key: str, filters: dict, vc_enabled: bool, use_sysname: bool = True, strip_domain: bool = False +) -> str: + """ + Generate a consistent cache metadata key from filter parameters. + + Args: + server_key: LibreNMS server identifier + filters: Filter dictionary + vc_enabled: Whether VC detection is enabled + use_sysname: Whether sysName is preferred over hostname for device naming + strip_domain: Whether domain suffix is stripped from device names + + Returns: + str: Consistent cache key for metadata + """ + # Sort filter items to ensure consistent key generation; use "is not None" to preserve + # valid falsy values like 0 and False (filtering only None/missing entries). + filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v is not None) + return f"librenms_filter_cache_metadata_{server_key}_{filter_parts}_{vc_enabled}_sysname={use_sysname}_strip={strip_domain}" + + +def get_active_cached_searches(server_key: str) -> list[dict]: + """ + Retrieve all active cached searches for a server and enrich with display-friendly values. + + Enriches raw filter IDs with human-readable names by looking up location names + from cached choices and converting type codes to display names. + + Args: + server_key: LibreNMS server identifier + + Returns: + List of dicts containing cache metadata with enriched display_filters + """ + from datetime import datetime, timezone + + cache_index_key = f"librenms_cache_index_{server_key}" + cache_index = cache.get(cache_index_key, []) + + active_searches = [] + valid_cache_keys = [] + + # Get location and type choices for enriching display + location_choices = {} + type_choices = { + "": "All Types", + "network": "Network", + "server": "Server", + "storage": "Storage", + "wireless": "Wireless", + "firewall": "Firewall", + "power": "Power", + "appliance": "Appliance", + "printer": "Printer", + "loadbalancer": "Load Balancer", + "other": "Other", + } + + # Get cached location choices for enrichment; scoped by server_key so labels + # from different LibreNMS servers don't bleed into each other's filter summaries. + location_cache_key = f"librenms_locations_choices:{server_key}" + cached_locations = cache.get(location_cache_key) + if cached_locations: + location_choices = dict(cached_locations) + + for cache_key in cache_index: + metadata = cache.get(cache_key) + if metadata: + # Cache still exists, calculate time remaining + cache_timeout = metadata.get("cache_timeout", 300) + now = datetime.now(timezone.utc) + try: + cached_at_raw = metadata.get("cached_at") + cached_at = ( + datetime.fromisoformat(cached_at_raw) if cached_at_raw else datetime.fromtimestamp(0, timezone.utc) + ) + # Normalize naive datetimes (e.g., stored without tzinfo) to UTC + if cached_at.tzinfo is None: + cached_at = cached_at.replace(tzinfo=timezone.utc) + except (ValueError, TypeError): + cached_at = datetime.fromtimestamp(0, timezone.utc) + age_seconds = (now - cached_at).total_seconds() + remaining_seconds = max(0, cache_timeout - age_seconds) + + if remaining_seconds > 0: + # Add remaining time and cache key + metadata["remaining_seconds"] = int(remaining_seconds) + metadata["cache_key"] = cache_key + + # Enrich filters with human-readable display values + if "filters" in metadata: + display_filters = metadata["filters"].copy() + # Convert location ID to location name + if "location" in display_filters and display_filters["location"] in location_choices: + display_filters["location"] = location_choices[display_filters["location"]] + # Convert type code to display name + if "type" in display_filters and display_filters["type"] in type_choices: + display_filters["type"] = type_choices[display_filters["type"]] + metadata["display_filters"] = display_filters + else: + # Fallback if filters key missing + metadata["display_filters"] = {} + + active_searches.append(metadata) + valid_cache_keys.append(cache_key) + + # Clean up index if any keys have expired + if len(valid_cache_keys) < len(cache_index): + cache.set(cache_index_key, valid_cache_keys, timeout=3600) + + # Sort by most recent first + active_searches.sort(key=lambda x: x.get("cached_at", ""), reverse=True) + + return active_searches + + +def get_validated_device_cache_key( + server_key: str, + filters: dict, + device_id: int | str, + vc_enabled: bool, + use_sysname: bool = True, + strip_domain: bool = False, +) -> str: + """ + Generate a consistent cache key for validated device data. + + This ensures both synchronous and background job processing use the same + cache keys, avoiding duplicate validation work and cache entries. + + Args: + server_key: LibreNMS server key + filters: Filter dict with location, type, os, hostname, sysname, hardware keys + device_id: LibreNMS device ID + vc_enabled: Whether virtual chassis detection was enabled + use_sysname: Whether sysName is preferred over hostname for device naming + strip_domain: Whether domain suffix is stripped from device names + + Returns: + str: Cache key for the validated device + + Example: + >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) + >>> key + 'validated_device_default_e3b0c44298fc1c14_123_vc' + """ + # Sort filters for a deterministic, cross-process stable hash + filter_hash = hashlib.sha256(json.dumps(sorted(filters.items()), sort_keys=True).encode()).hexdigest()[:16] + vc_part = "vc" if vc_enabled else "novc" + return ( + f"validated_device_{server_key}_{filter_hash}_{device_id}_{vc_part}_sysname={use_sysname}_strip={strip_domain}" + ) + + +def get_import_device_cache_key(device_id: int | str, server_key: str = "default") -> str: + """ + Generate cache key for raw LibreNMS device data. + + This key is used to cache raw device data (without validation metadata) + to avoid redundant API calls when users interact with dropdowns during + the import workflow. + + Args: + device_id: LibreNMS device ID + server_key: LibreNMS server identifier for multi-server setups + + Returns: + str: Cache key for the device data + + Example: + >>> get_import_device_cache_key(123, "production") + 'import_device_data_production_123' + """ + return f"import_device_data_{server_key}_{device_id}" diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py new file mode 100644 index 0000000000..487c915cf1 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -0,0 +1,966 @@ +"""Device validation, import, and fetch operations.""" + +import logging + +from dcim.models import Device, DeviceRole, DeviceType, Rack, Site +from django.core.cache import cache +from django.db import transaction +from django.db.models import Q +from django.utils import timezone +from virtualization.models import Cluster # noqa: F401 β€” used by test mock.patch targets + +from ..librenms_api import LibreNMSAPI +from ..utils import ( + find_matching_platform, + find_matching_site, + match_librenms_hardware_to_device_type, +) +from .cache import get_import_device_cache_key +from .virtual_chassis import ( + _generate_vc_member_name, + empty_virtual_chassis_data, + get_virtual_chassis_data, + update_vc_member_suggested_names, +) + +logger = logging.getLogger(__name__) + + +def _try_chassis_device_type_match(api, device_id): + """ + Attempt device type matching using chassis inventory fields. + + When the LibreNMS hardware string doesn't match any NetBox device type, + the chassis entity often contains a more standardized identifier + (e.g., entPhysicalName 'CHAS-BP-MX480-S' or entPhysicalModelName '710-017414') + that matches a DeviceType part_number or model. + + Tries entPhysicalName first (typically the chassis part number), + then entPhysicalModelName as fallback. + + Returns: + dict with matched/device_type/match_type keys, or None on failure. + """ + skip_values = {"", "-", "Unspecified", "BUILTIN", "None"} + + try: + success, inventory = api.get_inventory_filtered(device_id, ent_physical_class="chassis") + if not success or not inventory: + return None + + for item in inventory: + # Try entPhysicalName first (often the chassis part number like CHAS-BP-MX480-S) + for field in ("entPhysicalName", "entPhysicalModelName"): + value = item.get(field) or "" + if value and value not in skip_values: + chassis_match = match_librenms_hardware_to_device_type(value) + if chassis_match["matched"]: + chassis_match["match_type"] = "chassis" + chassis_match["chassis_model"] = value + return chassis_match + except Exception: + logger.debug(f"Chassis inventory fallback failed for device {device_id}", exc_info=True) + + return None + + +def _determine_device_name( + libre_device: dict, + use_sysname: bool = True, + strip_domain: bool = False, + device_id: int | str = None, +) -> str: + """ + Determine the device/VM name from LibreNMS data. + + Centralized logic for building device names with consistent handling of: + - sysName vs hostname preference + - Domain stripping (avoiding IP addresses) + - Fallback to device_id when name is missing + + Args: + libre_device: Device data from LibreNMS + use_sysname: If True, prefer sysName; if False, use hostname + strip_domain: If True, strip domain suffix (e.g., '.example.com') + device_id: LibreNMS device ID for fallback name generation + + Returns: + str: The determined device name + + Example: + >>> _determine_device_name({'sysName': 'router.example.com', 'hostname': 'router'}, + ... use_sysname=True, strip_domain=True) + 'router' + """ + # Determine base name based on use_sysname preference + if use_sysname: + name = libre_device.get("sysName") or libre_device.get("hostname") + else: + name = libre_device.get("hostname") or libre_device.get("sysName") + + # Fallback to device_id if no name found + if not name: + if device_id is not None: + name = f"device-{device_id}" + else: + name = libre_device.get("device_id", "unknown") + name = f"device-{name}" + + # Strip domain if requested (but not for IP addresses) + if strip_domain and name and "." in name: + try: + from ipaddress import ip_address + + ip_address(name) + # It's a valid IP address, don't strip + except ValueError: + # Not an IP, safe to strip domain + name = name.split(".")[0] + + return name + + +def validate_device_for_import( + libre_device: dict, + import_as_vm: bool = False, + api: "LibreNMSAPI" = None, + *, + include_vc_detection: bool = True, + force_vc_refresh: bool = False, + use_sysname: bool = True, + strip_domain: bool = False, + server_key: str = "default", +) -> dict: + """ + Validate if a LibreNMS device can be imported to NetBox. + + Performs comprehensive validation: + - Checks if device already exists in NetBox + - Validates required prerequisites (Site, DeviceType, DeviceRole for devices) + OR (Cluster for VMs) + - Provides smart matching for missing objects + - Detects virtual chassis/stack configuration (if API provided) + - Returns detailed validation status + + Args: + libre_device: Device data from LibreNMS + import_as_vm: If True, validate for VM import instead of device import + api: Optional LibreNMSAPI instance for virtual chassis detection + include_vc_detection: Skip VC detection when False to speed up bulk operations + force_vc_refresh: When True, bypass cached VC data and re-query LibreNMS + use_sysname: If True, prefer sysName over hostname (matches import behaviour) + strip_domain: If True, strip domain suffix from device name + + Returns: + dict: Validation result with structure: + { + 'is_ready': bool, # Can import without user intervention + 'can_import': bool, # Can import (possibly after configuration) + 'import_as_vm': bool, # Whether importing as VM + 'existing_device': Device or VirtualMachine or None, + 'issues': List[str], # Blocking issues + 'warnings': List[str], # Non-blocking warnings + 'site': { # Only for devices + 'found': bool, + 'site': Site or None, + 'match_type': str, # 'exact' or None + 'suggestions': List[Site] # Alternative suggestions + }, + 'device_type': { # Only for devices + 'found': bool, + 'device_type': DeviceType or None, + 'match_type': str, # 'exact' or None + 'suggestions': List[dict] # Device types for user selection + }, + 'device_role': { # Only for devices + 'found': bool, # Always False - requires manual selection + 'role': DeviceRole or None, + 'available_roles': List[DeviceRole] # All roles for user selection + }, + 'cluster': { # Only for VMs + 'found': bool, # Always False - requires manual selection + 'cluster': Cluster or None, + 'available_clusters': List[Cluster] # All clusters for user selection + }, + 'platform': { + 'found': bool, + 'platform': Platform or None, + 'match_type': str # 'exact' or None + } + } + + Example: + >>> validation = validate_device_for_import(libre_device) + >>> if validation['is_ready']: + ... import_single_device(libre_device['device_id']) + """ + result = { + "is_ready": False, + "can_import": False, + "import_as_vm": import_as_vm, + "resolved_name": None, # Final device name after applying user preferences + "existing_device": None, + "existing_match_type": None, # Track how existing device was matched + "serial_action": None, # None, "link", "conflict", "update_serial", "hostname_differs" + "serial_confirmed": False, # True when librenms_id match and serial matches + "serial_duplicate": False, # True when incoming serial is already on a different device + "librenms_id_needs_migration": False, # True when librenms_id is still a legacy bare int + "name_matches": False, # True when existing device name matches LibreNMS sysName + "name_sync_available": False, # True when existing device name differs from sysName + "suggested_name": None, # sysName to suggest when name_sync_available is True + "device_type_mismatch": False, # True when existing device's type differs from LibreNMS + "issues": [], + "warnings": [], + "virtual_chassis": empty_virtual_chassis_data(), + "site": { + "found": False, + "site": None, + "match_type": None, + "suggestions": [], + }, + "device_type": { + "found": False, + "device_type": None, + "match_type": None, + "suggestions": [], + }, + "device_role": { + "found": False, + "role": None, + "available_roles": [], + }, + "cluster": { + "found": False, + "cluster": None, + "available_clusters": [], + }, + "platform": {"found": False, "platform": None, "match_type": None}, + "rack": { + "found": False, + "rack": None, + "available_racks": [], + }, + "naming_criteria": None, # Populated after resolved_name is set + } + + try: + # 1. Check if device/VM already exists in NetBox + # Always check both Devices AND VMs to properly detect existing objects + librenms_id = libre_device.get("device_id") + hostname = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=strip_domain, + device_id=librenms_id, + ) + result["resolved_name"] = hostname + result["naming_criteria"] = { + "use_sysname": use_sysname, + "strip_domain": strip_domain, + "raw_sysname": libre_device.get("sysName") or "", + "raw_hostname": libre_device.get("hostname") or "", + "source": "sysname" if use_sysname and libre_device.get("sysName") else "hostname", + } + logger.debug( + f"Checking for existing device/VM: " + f"librenms_id={librenms_id} (type={type(librenms_id).__name__}), " + f"hostname={hostname}" + ) + + from virtualization.models import VirtualMachine + + # Check for existing VM first (by librenms_id custom field) + try: + from netbox_librenms_plugin.utils import find_by_librenms_id + + existing_vm = find_by_librenms_id(VirtualMachine, int(librenms_id), server_key) + except (ValueError, TypeError): + # librenms_id is not convertible to int; no match will be found + existing_vm = None + + if existing_vm: + logger.info(f"Found existing VM: {existing_vm.name} (matched by librenms_id={librenms_id})") + result["existing_device"] = existing_vm + result["existing_match_type"] = "librenms_id" + result["import_as_vm"] = True # Force VM mode since VM exists + result["can_import"] = False + + # Detect legacy bare-integer format so UI can offer a migration action. + # Direct access needed to detect legacy integer format for migration prompt: + # LibreNMSAPI.get_librenms_id() returns an int in both formats, so only the + # raw type check on custom_field_data reveals whether migration is needed. + if isinstance(existing_vm.custom_field_data.get("librenms_id"), int): + result["librenms_id_needs_migration"] = True + + # Check if name matches resolved name (accounts for use_sysname/strip_domain) + # Note: name_sync_available/suggested_name are intentionally not set for VMs + # because UpdateDeviceNameView only supports Device objects; VM name-sync + # would require a separate implementation. + if hostname and existing_vm.name == hostname: + result["name_matches"] = True + + # Check for existing Device (by librenms_id custom field) + if not result["existing_device"]: + try: + from netbox_librenms_plugin.utils import find_by_librenms_id + + existing_device = find_by_librenms_id(Device, int(librenms_id), server_key) + except (ValueError, TypeError): + # librenms_id is not convertible to int; no match will be found + existing_device = None + + if existing_device: + logger.info(f"Found existing device: {existing_device.name} (matched by librenms_id={librenms_id})") + result["existing_device"] = existing_device + result["existing_match_type"] = "librenms_id" + result["can_import"] = False + + # Detect legacy bare-integer format so UI can offer a migration action. + # Direct access needed to detect legacy integer format for migration prompt: + # LibreNMSAPI.get_librenms_id() returns an int in both formats, so only the + # raw type check on custom_field_data reveals whether migration is needed. + if isinstance(existing_device.custom_field_data.get("librenms_id"), int): + result["librenms_id_needs_migration"] = True + + # Check if name matches resolved name (VC-aware: compare against VC member name) + if hostname and existing_device.virtual_chassis and existing_device.vc_position: + vc_expected_name = _generate_vc_member_name( + hostname, + existing_device.vc_position, + serial=existing_device.serial or "", + ) + if existing_device.name == vc_expected_name: + result["name_matches"] = True + else: + result["name_sync_available"] = True + result["suggested_name"] = vc_expected_name + elif hostname and existing_device.name == hostname: + result["name_matches"] = True + elif hostname and existing_device.name != hostname: + result["name_sync_available"] = True + result["suggested_name"] = hostname + + # Check for serial drift on the linked device + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-": + if existing_device.serial and existing_device.serial == incoming_serial: + result["serial_confirmed"] = True + elif existing_device.serial and existing_device.serial != incoming_serial: + serial_conflict = ( + Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + ) + if serial_conflict: + result["serial_action"] = "conflict" + result["serial_duplicate"] = True + result["warnings"].append( + f"Serial conflict: incoming serial '{incoming_serial}' is already assigned to " + f"device '{serial_conflict.name}' (ID: {serial_conflict.pk}) in NetBox. " + f"Investigate which device should own this serial before updating." + ) + else: + result["serial_action"] = "update_serial" + result["warnings"].append( + f"Serial number differs (NetBox: '{existing_device.serial}', " + f"LibreNMS: '{incoming_serial}'). Hardware may have been replaced." + ) + + # Only check hostname/serial/IP if not already matched by librenms_id + if not result["existing_device"]: + # Check by hostname/name - Check both VMs and Devices for conflicts + existing_vm = VirtualMachine.objects.filter(name__iexact=hostname).first() + existing_device = Device.objects.filter(name__iexact=hostname).first() + + # If BOTH exist with same hostname, it's ambiguous - don't match either + if existing_vm and existing_device: + logger.warning( + f"Hostname conflict: Both VM '{existing_vm.name}' and Device " + f"'{existing_device.name}' exist with hostname '{hostname}'" + ) + result["warnings"].append( + f"Both a VM and Device exist with hostname '{hostname}' in NetBox. " + f"Cannot determine which to match. Please set the librenms_id custom field on the correct object." + ) + # Don't set existing_device, don't block import - let user proceed as new + # This allows them to import and then resolve the conflict manually + elif existing_vm: + logger.info(f"Found existing VM by hostname: {existing_vm.name}") + result["existing_device"] = existing_vm + result["existing_match_type"] = "hostname" + result["import_as_vm"] = True # Force VM mode since VM exists + result["warnings"].append( + f"VM with same hostname exists in NetBox as '{existing_vm.name}' (not linked to LibreNMS)" + ) + result["can_import"] = False + elif existing_device: + logger.info(f"Found existing device by hostname: {existing_device.name}") + result["existing_device"] = existing_device + result["existing_match_type"] = "hostname" + + # Check for serial conflict on hostname-matched device + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-" and existing_device.serial != incoming_serial: + serial_conflict = ( + Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + ) + if serial_conflict: + result["serial_action"] = "conflict" + result["serial_duplicate"] = True + result["warnings"].append( + f"Serial conflict: incoming serial '{incoming_serial}' is already assigned to " + f"device '{serial_conflict.name}' (ID: {serial_conflict.pk}) in NetBox. " + f"Investigate which device should own this serial before importing." + ) + else: + result["serial_action"] = "update_serial" + result["warnings"].append( + f"Hostname matches but serial differs (NetBox: '{existing_device.serial}', " + f"LibreNMS: '{incoming_serial}'). Hardware may have been replaced." + ) + else: + result["warnings"].append( + f"Device with same hostname exists in NetBox as '{existing_device.name}' (not linked to LibreNMS)" + ) + + result["can_import"] = False + + # Check by serial number (strong physical match - hardware identity) + if not result["existing_device"]: + serial = libre_device.get("serial") or "" + if serial and serial != "-" and not import_as_vm: + existing_by_serial = Device.objects.filter(serial=serial).first() + if existing_by_serial: + logger.info(f"Found existing device by serial: {existing_by_serial.name} (serial={serial})") + result["existing_device"] = existing_by_serial + result["existing_match_type"] = "serial" + result["can_import"] = False + + if existing_by_serial.name and existing_by_serial.name.lower() == hostname.lower(): + result["warnings"].append( + f"Device with same serial and hostname exists as '{existing_by_serial.name}' " + f"(not linked to LibreNMS)" + ) + result["serial_action"] = "link" + else: + result["warnings"].append( + f"Device with same serial ({serial}) exists as '{existing_by_serial.name}' " + f"but hostname differs (LibreNMS: '{hostname}'). Device may have been reinstalled." + ) + result["serial_action"] = "hostname_differs" + + # Check by primary IP (weaker match, IP could be reassigned) - only for devices + if not result["existing_device"]: + primary_ip = libre_device.get("ip") + if primary_ip and not import_as_vm: + from ipam.models import IPAddress + + existing_ip = IPAddress.objects.filter(address__net_host=primary_ip).first() + if existing_ip and existing_ip.assigned_object: + device = ( + existing_ip.assigned_object.device + if hasattr(existing_ip.assigned_object, "device") + else None + ) + if device: + result["existing_device"] = device + result["existing_match_type"] = "primary_ip" + result["warnings"].append( + f"IP address {primary_ip} already assigned to device '{device.name}' (not linked to LibreNMS)" + ) + result["can_import"] = False + + # Refresh local variable to reflect any VM-mode adjustments made during detection + # (e.g. existing VM found by hostname sets result["import_as_vm"] = True) + import_as_vm = result["import_as_vm"] + + # Validate based on import type (Device or VM) + if import_as_vm: + # 2. For VMs: Validate Cluster (required) - Must be manually selected + result["cluster"]["found"] = False + result["issues"].append("Cluster must be manually selected before importing as VM") + # Provide list of available clusters for user selection (cached) + cache_key = "librenms_import_all_clusters" + all_clusters = cache.get(cache_key) + if all_clusters is None: + all_clusters = list(Cluster.objects.all()) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, all_clusters, cache_timeout) + result["cluster"]["available_clusters"] = all_clusters + + # Skip device-specific validations for VMs + result["site"]["found"] = True # Not required for VMs + result["device_type"]["found"] = True # Not required for VMs + result["device_role"]["found"] = True # Not required for VMs + + else: + # 2. For Devices: Validate Site (required) + location = libre_device.get("location", "") + site_match = find_matching_site(location) + result["site"] = site_match + + if not site_match["found"]: + result["issues"].append(f"No matching site found for location: '{location}'") + # Get alternative suggestions + if location: + all_sites = Site.objects.all()[:10] # Limit for performance + result["site"]["suggestions"] = list(all_sites) + + # 3. Validate DeviceType (required) + hardware = libre_device.get("hardware", "") + dt_match = match_librenms_hardware_to_device_type(hardware) + + # Chassis inventory fallback: when hardware doesn't match, + # try the chassis entPhysicalModelName as an additional lookup source + if not dt_match["matched"] and api: + device_id = libre_device.get("device_id") + if device_id: + chassis_match = _try_chassis_device_type_match(api, device_id) + if chassis_match and chassis_match["matched"]: + dt_match = chassis_match + + # Update result keys individually to preserve the existing schema (especially "found") + result["device_type"]["found"] = dt_match["matched"] + result["device_type"]["device_type"] = dt_match.get("device_type") + result["device_type"]["match_type"] = dt_match.get("match_type") + + if not dt_match["matched"]: + result["device_type"]["found"] = False + result["issues"].append(f"No matching device type found for hardware: '{hardware}'") + # Get some device types for user to choose from + all_device_types = DeviceType.objects.all()[:10] + result["device_type"]["suggestions"] = [ + { + "device_type": dt, + "similarity": 0.0, # No fuzzy matching, just showing options + "match_field": None, + } + for dt in all_device_types + ] + + # 4. DeviceRole (required) - Must be manually selected by user + logger.debug(f"[{hostname}] Issues BEFORE adding role issue: {result['issues']}") + result["device_role"]["found"] = False + result["issues"].append("Device role must be manually selected before import") + logger.debug(f"[{hostname}] Issues AFTER adding role issue: {result['issues']}") + # Provide list of available roles for user selection (cached) + cache_key = "librenms_import_all_roles" + all_roles = cache.get(cache_key) + if all_roles is None: + all_roles = list(DeviceRole.objects.all()) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, all_roles, cache_timeout) + result["device_role"]["available_roles"] = all_roles + + # 4b. Rack (optional) - Provide available racks for the matched site + if site_match["found"] and site_match["site"]: + site = site_match["site"] + # Use cache to optimize rack lookups per site + cache_key = f"librenms_import_racks_site_{site.pk}" + available_racks = cache.get(cache_key) + + if available_racks is None: + # Query racks for this site - include both: + # 1. Racks assigned to locations within the site + # 2. Racks directly assigned to the site (without location) + available_racks = list( + Rack.objects.filter(Q(location__site=site) | Q(site=site)) + .select_related("location", "site") + .order_by("location__name", "name") + ) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, available_racks, cache_timeout) + + result["rack"]["available_racks"] = available_racks + # Rack is optional, don't add to issues + result["rack"]["found"] = True # Mark as "found" even if None (optional field) + + # Skip VM-specific validations for devices + result["cluster"]["found"] = True # Not required for devices + + # 5. Match Platform (optional - same for both devices and VMs) + os = libre_device.get("os", "") + platform_match = find_matching_platform(os) + result["platform"] = platform_match + + if not platform_match["found"] and os: + result["warnings"].append(f"No matching platform found for OS: '{os}'") + + # 6. Additional validations + if not hostname: + result["issues"].append("Device has no hostname") + + # 7. Virtual chassis detection (only for devices, not VMs) + if include_vc_detection and not import_as_vm and api is not None: + device_id = libre_device.get("device_id") + if device_id: + try: + logger.debug(f"Calling get_virtual_chassis_data for device {device_id}") + vc_detection = get_virtual_chassis_data(api, device_id, force_refresh=force_vc_refresh) + logger.debug( + f"VC detection result: is_stack={vc_detection.get('is_stack')}, " + f"member_count={vc_detection.get('member_count')}, " + f"members={len(vc_detection.get('members', []))}" + ) + if vc_detection: + result["virtual_chassis"] = vc_detection + if vc_detection["is_stack"]: + logger.debug( + f"Virtual chassis CONFIRMED for device {hostname}: " + f"{vc_detection['member_count']} members" + ) + result["virtual_chassis"] = update_vc_member_suggested_names(vc_detection, hostname) + except Exception as e: + logger.exception(f"Exception during VC detection for device {hostname}: {e}") + result["virtual_chassis"]["detection_error"] = str(e) + else: + logger.debug(f"No device_id found for {hostname}") + + # 8. Determine if device/VM is ready to import + if result["existing_device"]: + # Already matched - can_import was already set to False + result["is_ready"] = False + # Populate role from existing device so the modal shows it + existing = result["existing_device"] + if hasattr(existing, "role") and existing.role: + result["device_role"]["found"] = True + result["device_role"]["role"] = existing.role + + # Check for device type mismatch between existing device and LibreNMS + if hasattr(existing, "device_type") and existing.device_type: + librenms_dt = result["device_type"].get("device_type") + if librenms_dt and existing.device_type.pk != librenms_dt.pk: + result["device_type_mismatch"] = True + result["warnings"].append( + f"Device type mismatch: NetBox has '{existing.device_type}' " + f"but LibreNMS reports '{librenms_dt}'. " + f"This may indicate the wrong device was matched." + ) + else: + result["can_import"] = len(result["issues"]) == 0 + + if import_as_vm: + # For VMs: only cluster is required + result["is_ready"] = result["can_import"] and result["cluster"]["found"] + else: + # For Devices: site, device_type, and device_role are required + result["is_ready"] = ( + result["can_import"] + and result["site"]["found"] + and result["device_type"]["found"] + and result["device_role"]["found"] + ) + + logger.debug( + f"Validation for {libre_device.get('hostname')} ({'VM' if import_as_vm else 'Device'}): " + f"issues={len(result['issues'])}, can_import={result['can_import']}, " + f"issues_list={result['issues']}" + ) + + return result + + except Exception as e: + logger.exception(f"Error validating device for import: {libre_device.get('hostname', 'unknown')}") + result["issues"].append(f"Validation error: {str(e)}") + return result + + +def import_single_device( + device_id: int, + server_key: str = None, + validation: dict = None, + manual_mappings: dict = None, + sync_options: dict = None, + libre_device: dict = None, +) -> dict: + """ + Import a single LibreNMS device to NetBox. + + Args: + device_id: LibreNMS device ID + server_key: LibreNMS server configuration key + validation: Pre-computed validation dict (optional) + manual_mappings: Manual object mappings (optional): + - site_id: NetBox Site ID + - device_type_id: NetBox DeviceType ID + - device_role_id: NetBox DeviceRole ID + - platform_id: NetBox Platform ID (optional) + - rack_id: NetBox Rack ID (optional) + sync_options: Sync options (optional): + - sync_interfaces: bool (default True) + - sync_cables: bool (default True) + - sync_ips: bool (default True) + - sync_fields: bool (default True) + libre_device: Pre-fetched LibreNMS device data (optional). + If provided, skips API call to fetch device info. + + Returns: + dict: Import result with structure: + { + 'success': bool, + 'device': Device object or None, + 'message': str, + 'error': str or None, + 'synced': { + 'interfaces': int, + 'cables': int, + 'ip_addresses': int + } + } + """ + try: + api = LibreNMSAPI(server_key=server_key) + + # Use pre-fetched device data if provided, otherwise fetch from API + if libre_device is None: + success, libre_device = api.get_device_info(device_id) + if not success or not libre_device: + return { + "success": False, + "device": None, + "message": "", + "error": f"Failed to retrieve device {device_id} from LibreNMS", + "synced": {}, + } + + # Validate device if validation not provided + if validation is None: + use_sysname_opt = sync_options.get("use_sysname", True) if sync_options else True + strip_domain_opt = sync_options.get("strip_domain", False) if sync_options else False + validation = validate_device_for_import( + libre_device, + use_sysname=use_sysname_opt, + strip_domain=strip_domain_opt, + server_key=api.server_key, + ) + + # Check if device already exists + if validation.get("existing_device"): + return { + "success": False, + "device": validation["existing_device"], + "message": "", + "error": f"Device already exists: {validation['existing_device'].name}", + "synced": {}, + } + + # Use validation-derived matches, allow manual mappings to override specific fields + site = validation["site"].get("site") + device_type = validation["device_type"].get("device_type") + device_role = validation["device_role"].get("role") + platform = validation["platform"].get("platform") + rack = validation.get("rack", {}).get("rack") + + if manual_mappings: + site = Site.objects.filter(id=manual_mappings.get("site_id")).first() or site + device_type = DeviceType.objects.filter(id=manual_mappings.get("device_type_id")).first() or device_type + device_role = DeviceRole.objects.filter(id=manual_mappings.get("device_role_id")).first() or device_role + + platform_id = manual_mappings.get("platform_id") + if platform_id: + from dcim.models import Platform + + platform = Platform.objects.filter(id=platform_id).first() or platform + + rack_id = manual_mappings.get("rack_id") + if rack_id: + rack = Rack.objects.select_related("location", "site").filter(id=rack_id).first() or rack + + rack = rack or validation.get("rack", {}).get("rack") + + # Validate required fields + if not site: + return { + "success": False, + "device": None, + "message": "", + "error": "Site is required but not provided", + "synced": {}, + } + if not device_type: + return { + "success": False, + "device": None, + "message": "", + "error": "Device type is required but not provided", + "synced": {}, + } + if not device_role: + return { + "success": False, + "device": None, + "message": "", + "error": "Device role is required but not provided", + "synced": {}, + } + + # Create device in NetBox + with transaction.atomic(): + # Determine device name based on sync options + use_sysname = sync_options.get("use_sysname", True) if sync_options else True + strip_domain = sync_options.get("strip_domain", False) if sync_options else False + + device_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=strip_domain, + device_id=device_id, + ) + + # Generate import timestamp comment + import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") + + device_data = { + "name": device_name, + "site": site, + "device_type": device_type, + "role": device_role, + "status": "active" if libre_device.get("status") == 1 else "offline", + "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", + "custom_field_data": {"librenms_id": {api.server_key: int(device_id)}}, + } + + # Add optional fields + if platform: + device_data["platform"] = platform + + if rack: + device_data["rack"] = rack + + serial = libre_device.get("serial", "") + if serial and serial != "-": + device_data["serial"] = serial + + location_name = libre_device.get("location", "") + if location_name and location_name != "-": + from dcim.models import Location + + # Try to find matching location within the site + location = Location.objects.filter(site=site, name__iexact=location_name).first() + if location: + device_data["location"] = location + + # Create the device + device = Device(**device_data) + device.full_clean() + device.save() + + # Sync additional data based on options + sync_options = sync_options or {} + synced = {"interfaces": 0, "cables": 0, "ip_addresses": 0} + + try: + # Sync interfaces + if sync_options.get("sync_interfaces", True): + # This is simplified - would need proper request context + # For now, just log that it should be done + logger.info(f"Interface sync should be performed for device {device.name}") + + # Sync cables + if sync_options.get("sync_cables", True): + logger.info(f"Cable sync should be performed for device {device.name}") + + # Sync IP addresses + if sync_options.get("sync_ips", True): + logger.info(f"IP address sync should be performed for device {device.name}") + + except Exception as e: + logger.warning(f"Error during post-import sync: {str(e)}") + # Don't fail the import if sync fails + + return { + "success": True, + "device": device, + "message": f"Successfully imported device: {device.name}", + "error": None, + "synced": synced, + } + + except Exception as e: + logger.exception(f"Error importing device {device_id}") + return { + "success": False, + "device": None, + "message": "", + "error": str(e), + "synced": {}, + } + + +def get_librenms_device_by_id(api: LibreNMSAPI, device_id: int) -> dict: + """ + Retrieve a single device from LibreNMS by ID. + + Args: + api: LibreNMSAPI instance + device_id: LibreNMS device ID + + Returns: + Device dictionary or None if not found + """ + try: + # Use the dedicated API endpoint to get device by ID + success, device = api.get_device_info(device_id) + if success and device: + return device + + logger.warning(f"Device {device_id} not found in LibreNMS") + return None + except Exception as e: + logger.exception(f"Failed to get device {device_id} from LibreNMS: {e}") + return None + + +def fetch_device_with_cache( + device_id: int, + api: LibreNMSAPI, + server_key: str = None, + libre_devices_cache: dict = None, +) -> dict | None: + """ + Fetch LibreNMS device from cache or API with automatic caching. + + Checks three sources in order: + 1. Pre-fetched cache dict (if provided) + 2. Django cache (Redis/memory) + 3. LibreNMS API (caches result for future use) + + This function consolidates the device fetching pattern used throughout + the import workflow, eliminating code duplication. + + Args: + device_id: LibreNMS device ID to fetch + api: LibreNMSAPI instance for fallback API calls + server_key: Optional server key for multi-server setups (defaults to api.server_key) + libre_devices_cache: Optional pre-fetched device cache dict + + Returns: + Device dict from LibreNMS, or None if not found + + Example: + >>> # Simple usage + >>> libre_device = fetch_device_with_cache(123, api) + >>> if libre_device: + ... print(libre_device['hostname']) + >>> + >>> # With pre-fetched cache dict + >>> cache_dict = {123: {...}, 456: {...}} + >>> libre_device = fetch_device_with_cache(123, api, libre_devices_cache=cache_dict) + """ + # Check pre-fetched cache dict first (fastest) + if libre_devices_cache and device_id in libre_devices_cache: + return libre_devices_cache[device_id] + + # Check Django cache + cache_key = get_import_device_cache_key(device_id, server_key or api.server_key) + libre_device = cache.get(cache_key) + + if not libre_device: + # Fallback to API fetch + libre_device = get_librenms_device_by_id(api, device_id) + if libre_device: + # Cache for future use + cache.set(cache_key, libre_device, timeout=api.cache_timeout) + + return libre_device diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py new file mode 100644 index 0000000000..3e12658066 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -0,0 +1,268 @@ +"""Device filtering and retrieval from LibreNMS.""" + +import hashlib +import json +import logging +from typing import List + +from django.core.cache import cache + +from ..librenms_api import LibreNMSAPI + +logger = logging.getLogger(__name__) + + +def get_device_count_for_filters( + api: LibreNMSAPI, + filters: dict, + clear_cache: bool = False, + show_disabled: bool = True, +) -> int: + """ + Get count of LibreNMS devices matching filters. + + This is a lightweight function to determine device count for background job + decision making. Uses the same caching as get_librenms_devices_for_import(). + + Args: + api: LibreNMS API client instance + filters: Filter dict with location, type, os, hostname, sysname keys + clear_cache: Whether to force cache refresh + show_disabled: Whether to include disabled devices + + Returns: + int: Count of devices matching filters + """ + devices = get_librenms_devices_for_import(api, filters=filters, force_refresh=clear_cache) + + # Filter out disabled devices if requested. LibreNMS's "disabled" field (1=disabled, + # 0=enabled) reflects manual device disablement; "status" reflects SNMP reachability. + # show_disabled controls the former: hidden when disabled==1, shown regardless of status. + if not show_disabled: + devices = [d for d in devices if int(d.get("disabled", 0)) != 1] + + return len(devices) + + +def get_librenms_devices_for_import( + api: LibreNMSAPI = None, + filters: dict = None, + server_key: str = None, + *, + force_refresh: bool = False, + return_cache_status: bool = False, +) -> List[dict] | tuple[List[dict], bool]: + """ + Retrieve LibreNMS devices based on filters. + + Args: + api: LibreNMSAPI instance (if not provided, creates one with server_key) + filters: Dict containing filter parameters: + - location: LibreNMS location/site filter + - type: Device type filter + - os: Operating system filter + - hostname: Hostname filter (partial match) + - sysname: System name filter (partial match) + - status: Device status filter (1=up, 0=down) + - disabled: Include disabled devices (0=active only, 1=all) + server_key: Key for specific server configuration (used if api not provided) + force_refresh: When True, bypass the cache and fetch fresh data + return_cache_status: When True, returns (devices, from_cache) tuple + + Returns: + List of device dictionaries from LibreNMS, or tuple of (devices, from_cache) + if return_cache_status is True. from_cache=True means data was loaded from + existing cache; from_cache=False means data was just fetched from LibreNMS. + """ + try: + # Use provided API instance or create a new one + if api is None: + api = LibreNMSAPI(server_key=server_key) + + # Build LibreNMS API filters using the type/query format + # LibreNMS API v0 expects ?type=X&query=Y format, not direct parameters + # NOTE: API only supports ONE type/query pair, so we'll use the most + # specific filter for the API and apply others client-side + api_filters = {} + client_filters = {} # Filters to apply after fetching from API + + if filters: + # Check for status filter first - it has special handling + if filters.get("status") is not None: + # Normalize to int: form fields send strings ("1"/"0"), API may send ints + try: + status_val = int(filters["status"]) + except (ValueError, TypeError): + status_val = None + # Status filter uses special types that don't need query param + if status_val == 1: + api_filters["type"] = "up" + elif status_val == 0: + api_filters["type"] = "down" + + # Save ALL other filters for client-side filtering when status is used + if filters.get("location"): + client_filters["location"] = filters["location"] + if filters.get("type"): + client_filters["type"] = filters["type"] + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + else: + # Priority order for type/query filters: location > type > os > hostname > sysname + # Note: When sysname is combined with other filters, it's applied client-side for partial matching + # When sysname is alone, it uses API exact match (type=sysName) + # Note: hardware is always applied client-side for partial matching + # Use first available for API, save others for client-side filtering + if filters.get("location"): + api_filters["type"] = "location_id" + api_filters["query"] = filters["location"] + # Save remaining filters for client-side + if filters.get("type"): + client_filters["type"] = filters["type"] + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("type"): + api_filters["type"] = "type" + api_filters["query"] = filters["type"] + # Save remaining filters for client-side + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("os"): + api_filters["type"] = "os" + api_filters["query"] = filters["os"] + # Save remaining filters for client-side + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("hostname"): + api_filters["type"] = "hostname" + api_filters["query"] = filters["hostname"] + # Save sysname and hardware for client-side + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("sysname"): + # sysname-only filter: Use API exact match (type=sysName&query=) + # This is safe - returns empty if no exact match found + api_filters["type"] = "sysName" + api_filters["query"] = filters["sysname"] + # Save hardware for client-side + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("hardware"): + # hardware-only filter: apply client-side for partial matching + client_filters["hardware"] = filters["hardware"] + + # Note: disabled filter isn't directly supported by LibreNMS API + # We'll filter client-side if needed + + # Use caching to avoid repeated API calls + # Include both API and client filters in cache key (deterministic, cross-process stable). + # Use api.server_key (always resolved) rather than the raw server_key arg (may differ). + def _hash(d): + return hashlib.sha256( + json.dumps(sorted(d.items()) if isinstance(d, dict) else d, sort_keys=True).encode() + ).hexdigest()[:16] + + cache_key = f"librenms_devices_import_{api.server_key}_{_hash(api_filters)}_{_hash(client_filters)}" + from_cache = False + + if force_refresh: + cache.delete(cache_key) + else: + cached_result = cache.get(cache_key) + if cached_result is not None: + # No need to deepcopy - cached data isn't mutated + devices = cached_result + from_cache = True + if return_cache_status: + return devices, from_cache + return devices + + success, devices = api.list_devices(api_filters if api_filters else None) + + if not success: + logger.error(f"Failed to retrieve devices from LibreNMS: {devices}") + if return_cache_status: + return [], False + return [] + + # Apply client-side filters if any + if client_filters: + devices = _apply_client_filters(devices, client_filters) + + # Cache using configured timeout (default 300s) + # No need to deepcopy - Django's cache backend handles serialization + cache.set(cache_key, devices, timeout=api.cache_timeout) + + if return_cache_status: + return devices, from_cache + return devices + + except Exception: + logger.exception("Error retrieving LibreNMS devices for import") + if return_cache_status: + return [], False + return [] + + +def _apply_client_filters(devices: List[dict], filters: dict) -> List[dict]: + """ + Apply client-side filters to device list. + + Args: + devices: List of device dicts from LibreNMS + filters: Dict of filters to apply (location, type, os, hostname, sysname) + + Returns: + Filtered list of devices + """ + filtered = devices + + if filters.get("location"): + location_id = str(filters["location"]) + filtered = [d for d in filtered if str(d.get("location_id", "")) == location_id] + + if filters.get("type"): + device_type = filters["type"].lower() + filtered = [d for d in filtered if d.get("type", "").lower() == device_type] + + if filters.get("os"): + os_filter = filters["os"].lower() + filtered = [d for d in filtered if os_filter in d.get("os", "").lower()] + + if filters.get("hostname"): + hostname_filter = filters["hostname"].lower() + filtered = [d for d in filtered if hostname_filter in d.get("hostname", "").lower()] + + if filters.get("sysname"): + sysname_filter = filters["sysname"].lower() + filtered = [d for d in filtered if sysname_filter in d.get("sysName", "").lower()] + + if filters.get("hardware"): + hardware_filter = filters["hardware"].lower() + filtered = [d for d in filtered if hardware_filter in (d.get("hardware") or "").lower()] + + return filtered diff --git a/netbox_librenms_plugin/import_utils/permissions.py b/netbox_librenms_plugin/import_utils/permissions.py new file mode 100644 index 0000000000..742e50893c --- /dev/null +++ b/netbox_librenms_plugin/import_utils/permissions.py @@ -0,0 +1,44 @@ +"""Permission check helpers for device import operations.""" + +from django.core.exceptions import PermissionDenied + + +def check_user_permissions(user, permissions): + """ + Check if user has all required permissions. + + Args: + user: The user object to check permissions for + permissions: List of permission strings (e.g., ['dcim.add_device', 'dcim.add_interface']) + + Returns: + tuple: (has_all_permissions: bool, missing_permissions: list[str]) + + Raises: + PermissionDenied: If user is None (no user context available) + """ + if user is None: + raise PermissionDenied("No user context available for permission check") + + missing = [perm for perm in permissions if not user.has_perm(perm)] + return (len(missing) == 0, missing) + + +def require_permissions(user, permissions, action_description="perform this action"): + """ + Require user has all permissions, raising PermissionDenied if not. + + Args: + user: The user object to check permissions for + permissions: List of permission strings + action_description: Human-readable description for error message + + Raises: + PermissionDenied: If user lacks any required permission + """ + has_perms, missing = check_user_permissions(user, permissions) + if not has_perms: + missing_str = ", ".join(missing) + raise PermissionDenied( + f"You do not have permission to {action_description}. Missing permissions: {missing_str}" + ) diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py new file mode 100644 index 0000000000..b95f32eaec --- /dev/null +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -0,0 +1,470 @@ +"""Virtual chassis detection, creation, and management.""" + +import logging +from typing import List + +from dcim.models import Device, VirtualChassis +from django.core.cache import cache +from django.db import transaction + +from ..librenms_api import LibreNMSAPI + +logger = logging.getLogger(__name__) + + +def empty_virtual_chassis_data() -> dict: + """Public helper for callers that need a blank VC payload.""" + + return { + "is_stack": False, + "member_count": 0, + "members": [], + "detection_error": None, + } + + +def _clone_virtual_chassis_data(data: dict | None) -> dict: + """Return a defensive copy of cached VC data to avoid shared references.""" + + if not data: + return empty_virtual_chassis_data() + + members = [] + for idx, member in enumerate(data.get("members", [])): + member_copy = member.copy() + raw_position = member_copy.get("position", idx + 1) + try: + member_copy["position"] = int(raw_position) + except (TypeError, ValueError): + member_copy["position"] = idx + 1 # 1-based fallback; position 0 is invalid + members.append(member_copy) + + member_count = data.get("member_count") or len(members) + + return { + "is_stack": bool(data.get("is_stack")), + "member_count": member_count, + "members": members, + "detection_error": data.get("detection_error"), + } + + +_VC_CACHE_VERSION = "v1" + + +def _vc_cache_key(api: LibreNMSAPI, device_id: int | str) -> str: + server_key = getattr(api, "server_key", "default") + return f"librenms_vc_detection_{_VC_CACHE_VERSION}_{server_key}_{device_id}" + + +def get_virtual_chassis_data(api: LibreNMSAPI, device_id: int | str, *, force_refresh: bool = False) -> dict: + """Fetch (and cache) virtual chassis data for a LibreNMS device.""" + + if not api or device_id is None: + return empty_virtual_chassis_data() + + cache_key = _vc_cache_key(api, device_id) + if not force_refresh: + cached = cache.get(cache_key) + if cached is not None: + return _clone_virtual_chassis_data(cached) + + detection_data = detect_virtual_chassis_from_inventory(api, device_id) + if detection_data and "detection_error" not in detection_data: + detection_data["detection_error"] = None + + cache_value = _clone_virtual_chassis_data(detection_data) if detection_data else empty_virtual_chassis_data() + + cache_timeout = getattr(api, "cache_timeout", 300) or 300 + cache.set(cache_key, cache_value, timeout=cache_timeout) + return _clone_virtual_chassis_data(cache_value) + + +def prefetch_vc_data_for_devices(api: LibreNMSAPI, device_ids: List[int], *, force_refresh: bool = False) -> None: + """ + Pre-warm the virtual chassis cache for multiple devices. + + This eliminates the 0.5-1s delay when rendering the import table + by proactively fetching VC data before validation. + + Args: + api: LibreNMSAPI instance + device_ids: List of LibreNMS device IDs to prefetch VC data for + force_refresh: When True, bypass cache and fetch fresh data + + Example: + >>> # Before rendering import table + >>> prefetch_vc_data_for_devices(api, [123, 124, 125]) + >>> # Now all validate_device_for_import() calls hit cache instantly + """ + if not api or not device_ids: + return + + logger.debug(f"Pre-warming VC cache for {len(device_ids)} devices") + + for idx, device_id in enumerate(device_ids): + # This populates the cache if empty, or skips if already cached + try: + get_virtual_chassis_data(api, device_id, force_refresh=force_refresh) + except (BrokenPipeError, ConnectionError, IOError, OSError) as e: + logger.warning(f"Connection error during VC prefetch at device {idx}: {e}") + # Stop processing if connection is broken + return + except Exception as e: + # Log but continue for other errors + logger.warning(f"Error prefetching VC data for device {device_id}: {e}") + + logger.debug(f"VC cache warming complete for {len(device_ids)} devices") + + +def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> dict: + """ + Detect if device is a stack/Virtual Chassis by analyzing ENTITY-MIB inventory. + Vendor-agnostic using standard hierarchical structure. + + Args: + api: LibreNMSAPI instance + device_id: LibreNMS device ID + + Returns: + dict with structure: + { + 'is_stack': bool, + 'member_count': int, + 'members': [ + { + 'serial': str, + 'position': int, + 'model': str, + 'name': str, + 'index': int, + 'description': str, + 'suggested_name': str # Generated using master device name + } + ] + } + Returns None if not a stack or detection fails. + + Detection Logic: + 1. Check root level (entPhysicalContainedIn=0) for parent container + 2. Find parent index (entPhysicalClass='stack' or 'chassis') + 3. Get children chassis at that parent's index + 4. If multiple chassis found -> Stack detected + """ + try: + # Get the master device info to use for naming + success, device_info = api.get_device_info(device_id) + master_name = None + if success and device_info: + master_name = device_info.get("sysName") or device_info.get("hostname") + + # Step 1: Get root level items + success, root_items = api.get_inventory_filtered(device_id, ent_physical_contained_in=0) + + if not success or not root_items: + logger.debug(f"No root inventory items found for device {device_id}") + return None + + # Step 2: Find parent container index + # Could be class="stack" or the main "chassis" + parent_index = None + for item in root_items: + item_class = item.get("entPhysicalClass") + if item_class in ["stack", "chassis"]: + parent_index = item.get("entPhysicalIndex") + logger.debug(f"VC detection: Found parent container at index {parent_index} for device {device_id}") + break + + if parent_index is None: + return None + + # Step 3: Get children chassis at next level + success, child_items = api.get_inventory_filtered( + device_id, + ent_physical_class="chassis", + ent_physical_contained_in=parent_index, + ) + + if not success: + return None + + # Filter for chassis only (in case API filter didn't work) + chassis_items = [item for item in (child_items or []) if item.get("entPhysicalClass") == "chassis"] + + # Step 4: Multiple chassis = stack + if len(chassis_items) <= 1: + return None + + # Step 5: Extract member info + members = [] + for idx, chassis in enumerate(chassis_items): + # entPhysicalParentRelPos is 1-based; fall back to idx+1 (not idx) so + # position 0 is never produced β€” VC positions must be β‰₯ 1. + raw_position = chassis.get("entPhysicalParentRelPos", idx + 1) + try: + position = int(raw_position) + except (TypeError, ValueError): + position = idx + 1 + member_data = { + "serial": chassis.get("entPhysicalSerialNum", ""), + "position": position, + "model": chassis.get("entPhysicalModelName", ""), + "name": chassis.get("entPhysicalName", ""), + "index": chassis.get("entPhysicalIndex"), + "description": chassis.get("entPhysicalDescr", ""), + } + + # Generate suggested name if we have master name. + # position is already 1-based, so pass it directly (no +1). + if master_name: + member_data["suggested_name"] = _generate_vc_member_name(master_name, position) + else: + member_data["suggested_name"] = f"Member-{position}" + + members.append(member_data) + + # Sort by position + members.sort(key=lambda m: m["position"]) + + logger.info(f"Detected stack with {len(members)} members for device {device_id}") + + return {"is_stack": True, "member_count": len(members), "members": members} + + except Exception as e: + logger.exception(f"Error detecting virtual chassis for device {device_id}: {e}") + return None + + +def _load_vc_member_name_pattern() -> str: + """Load the VC member name pattern from settings, with fallback to default.""" + from ..models import LibreNMSSettings + + try: + settings = LibreNMSSettings.objects.first() + return settings.vc_member_name_pattern if settings else "-M{position}" + except Exception as e: + logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") + return "-M{position}" + + +def _generate_vc_member_name(master_name: str, position: int, serial: str = None, pattern: str = None) -> str: + """ + Generate name for VC member device using configured pattern from settings. + + Args: + master_name: Name of the master/primary device + position: VC position number + serial: Optional serial number of the member device + pattern: Optional pre-loaded name pattern; if None, loaded from settings. + Pass a pre-loaded pattern when calling inside a loop to avoid + repeated DB queries. + + Returns: + Generated member device name + + Examples: + pattern="-M{position}" -> "switch01-M2" + pattern=" ({position})" -> "switch01 (2)" + pattern="-SW{position}" -> "switch01-SW2" + pattern=" [{serial}]" -> "switch01 [ABC123]" + """ + if pattern is None: + pattern = _load_vc_member_name_pattern() + + # Prepare format variables + format_vars = { + "master_name": master_name, + "position": position, + "serial": serial or "", + } + + # Apply pattern - pattern should be suffix/prefix, not full name + try: + formatted_suffix = pattern.format(**format_vars) + return f"{master_name}{formatted_suffix}" + except KeyError as e: + logger.error(f"Invalid placeholder in VC naming pattern '{pattern}': {e}. Using default.") + return f"{master_name}-M{position}" + + +def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: + """ + Regenerate suggested VC member names using the actual master device name. + + This ensures preview shows accurate names after use_sysname and strip_domain + are applied to the master device name. + + Args: + vc_data: Virtual chassis detection data dict + master_name: The actual name that will be used for master device in NetBox + + Returns: + Updated vc_data dict with corrected suggested_name for each member + """ + if not vc_data or not vc_data.get("is_stack"): + return vc_data + + # Load naming pattern once to avoid a DB query per member + vc_pattern = _load_vc_member_name_pattern() + for idx, member in enumerate(vc_data.get("members", [])): + raw_position = member.get("position", idx) + try: + base_position = int(raw_position) + except (TypeError, ValueError): + base_position = idx + position = base_position + 1 # Convert to 1-based position + member["position"] = base_position + member["suggested_name"] = _generate_vc_member_name( + master_name, position, serial=member.get("serial"), pattern=vc_pattern + ) + + return vc_data + + +def create_virtual_chassis_with_members(master_device: Device, members_info: list, libre_device: dict): + """ + Create Virtual Chassis and member devices from detection info. + + This function creates a NetBox VirtualChassis with the master device + and all detected member devices, wrapped in a transaction for safety. + + Args: + master_device: The imported device (becomes VC master) + members_info: List of member dicts from VC detection + libre_device: Original LibreNMS device data + + Returns: + VirtualChassis: The created virtual chassis instance + + Raises: + ValidationError: If member count validation fails + IntegrityError: If duplicate serials/names are detected + Exception: For other creation errors + + Example members_info: + [ + {'serial': 'ABC123', 'position': 0, 'model': 'C9300-48U', 'name': 'Switch 1'}, + {'serial': 'ABC124', 'position': 1, 'model': 'C9300-48U', 'name': 'Switch 2'} + ] + """ + + # original_master_name is still referenced in warning messages inside the atomic block. + original_master_name = master_device.name + + try: + with transaction.atomic(): + # Rename master device to include position 1 pattern + master_device_new_name = _generate_vc_member_name(original_master_name, 1, serial=master_device.serial) + + # Check if renamed master conflicts with existing device + if Device.objects.filter(name=master_device_new_name).exclude(pk=master_device.pk).exists(): + logger.warning( + f"Cannot rename master to '{master_device_new_name}' - name already exists. " + f"Keeping original name '{original_master_name}'" + ) + master_base_name = original_master_name + else: + master_device.name = master_device_new_name + master_base_name = original_master_name + + # Create VC using original base name + vc_name = master_base_name + vc = VirtualChassis.objects.create( + name=vc_name, + master=master_device, + domain=f"librenms-{libre_device.get('device_id', master_device.pk)}", + ) + + # Update master device + master_device.virtual_chassis = vc + master_device.vc_position = 1 # Master is position 1 + master_device.save() + + # Create member devices for remaining positions + position = 2 # Start at 2 (master is 1) + members_created = 0 + # Load naming pattern once to avoid a DB query per member + vc_pattern = _load_vc_member_name_pattern() + + for member in members_info: + # Skip if this is the master's serial (only when both serials are non-empty) + if member.get("serial") and member.get("serial") == master_device.serial: + continue + + serial = member.get("serial") + + member_rack = master_device.rack + member_location = master_device.location or ( + member_rack.location if member_rack and member_rack.location else None + ) + + # Check for duplicate serial + if serial and Device.objects.filter(serial=serial).exists(): + logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") + continue + + # Prefer the discovered SNMP position; fall back to sequential counter. + # Normalize discovered_pos: 0 is not a valid VC position, treat as absent. + try: + discovered_pos = int(member.get("position")) if member.get("position") is not None else None + except (TypeError, ValueError): + discovered_pos = None + if discovered_pos is not None and discovered_pos < 1: + discovered_pos = None # 0 is invalid for vc_position; fall back to counter + chosen_pos = discovered_pos if discovered_pos is not None else position + # Advance the sequential counter: + # - if discovered_pos was used, advance counter past it to avoid future reuse; + # - if counter was consumed as fallback, increment it normally. + if discovered_pos is None: + position += 1 + else: + position = max(position, discovered_pos + 1) + + member_name = _generate_vc_member_name(master_base_name, chosen_pos, serial=serial, pattern=vc_pattern) + + # Check for duplicate name + if Device.objects.filter(name=member_name).exists(): + logger.warning(f"Device with name '{member_name}' already exists, skipping VC member creation") + continue + + Device.objects.create( + name=member_name, + device_type=master_device.device_type, + role=master_device.role, + site=master_device.site, + location=member_location, + rack=member_rack, + platform=master_device.platform, + serial=serial, + virtual_chassis=vc, + vc_position=chosen_pos, + comments=f"VC member (LibreNMS: {member.get('name', 'Unknown')})\n" + f"Auto-created from stack inventory", + ) + members_created += 1 + + # Validate member count + expected_members = len( + [m for m in members_info if not (m.get("serial") and m.get("serial") == master_device.serial)] + ) + if members_created < expected_members: + logger.warning( + f"Created {members_created} members but expected {expected_members}. " + "Some members may have been skipped due to duplicates." + ) + + logger.info( + f"Created Virtual Chassis '{vc.name}' with {vc.members.count()} total members " + f"(1 master + {members_created} additional)" + ) + + return vc + + except Exception as e: + # The transaction.atomic() block above will roll back all DB changes automatically. + # Manual state restoration is redundant and the save() would fail in a broken transaction. + logger.error( + f"Virtual Chassis creation failed for device {master_device.name}: {e}", + exc_info=True, + ) + raise diff --git a/netbox_librenms_plugin/import_utils/vm_operations.py b/netbox_librenms_plugin/import_utils/vm_operations.py new file mode 100644 index 0000000000..1272871a5a --- /dev/null +++ b/netbox_librenms_plugin/import_utils/vm_operations.py @@ -0,0 +1,229 @@ +"""Virtual machine creation and import operations.""" + +import logging + +from dcim.models import DeviceRole +from django.utils import timezone +from virtualization.models import Cluster + +from ..librenms_api import LibreNMSAPI +from .device_operations import _determine_device_name, fetch_device_with_cache, validate_device_for_import +from .permissions import require_permissions + +logger = logging.getLogger(__name__) + + +def create_vm_from_librenms( + libre_device: dict, validation: dict, use_sysname: bool = True, role=None, server_key: str = "default" +): + """ + Create a NetBox VirtualMachine from LibreNMS device data. + + Args: + libre_device: Device data from LibreNMS + validation: Validation result from validate_device_for_import with import_as_vm=True + use_sysname: If True, prefer sysName; if False, use hostname + role: Optional DeviceRole to assign to the VM + + Returns: + Created VirtualMachine instance + + Raises: + Exception if VM cannot be created + """ + from virtualization.models import VirtualMachine + + if not validation["can_import"]: + raise ValueError(f"VM cannot be imported: {', '.join(validation['issues'])}") + + # Extract matched objects from validation + cluster = validation["cluster"]["cluster"] + platform = validation["platform"].get("platform") + + # Determine VM name - use pre-computed name if available (handles strip_domain) + vm_name = libre_device.get("_computed_name") + if not vm_name: + vm_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=False, + device_id=libre_device.get("device_id"), + ) + + # Generate import timestamp comment + import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") + + # Create the VM with librenms_id custom field + vm = VirtualMachine.objects.create( + name=vm_name, + cluster=cluster, + role=role, # Optional VM role + platform=platform, + comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", + custom_field_data={"librenms_id": {server_key: int(libre_device["device_id"])}}, + ) + + logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") + return vm + + +def bulk_import_vms( + vm_imports: dict[int, dict[str, int]], + api: LibreNMSAPI, + sync_options: dict = None, + libre_devices_cache: dict = None, + job=None, + user=None, +) -> dict: + """ + Import multiple LibreNMS devices as VMs in NetBox. + + Handles validation, cluster/role assignment, name determination, + and VM creation. Supports both synchronous and background job execution. + + This function consolidates VM import logic that was previously duplicated + in BulkImportDevicesView and ImportDevicesJob, ensuring consistent behavior + across synchronous and background import paths. + + Args: + vm_imports: Dict mapping device_id to {"cluster_id": int, "device_role_id": int} + api: LibreNMSAPI instance for device fetching + sync_options: Optional dict with use_sysname, strip_domain settings + libre_devices_cache: Optional pre-fetched device data cache + job: Optional JobRunner instance for background job logging/cancellation + user: User performing the import (for permission checks). If job is provided, + user is extracted from job.job.user if not explicitly passed. + + Returns: + Dict with keys: + - success: List of {"device_id": int, "device": VM, "message": str} + - failed: List of {"device_id": int, "error": str} + - skipped: List of {"device_id": int, "reason": str} + + Raises: + PermissionDenied: If user lacks required permissions + + Example: + >>> # Synchronous import from view + >>> vm_imports = {123: {"cluster_id": 5, "device_role_id": 2}} + >>> result = bulk_import_vms(vm_imports, api, sync_options, user=request.user) + >>> print(f"Created {len(result['success'])} VMs") + >>> + >>> # Background job import + >>> result = bulk_import_vms(vm_imports, api, sync_options, cache, job=self) + """ + from netbox_librenms_plugin.import_validation_helpers import ( + apply_cluster_to_validation, + apply_role_to_validation, + ) + + # Extract user from job if not explicitly provided + if user is None and job is not None: + user = getattr(job.job, "user", None) + + # Check permissions at start of bulk operation + require_permissions(user, ["virtualization.add_virtualmachine"], "import VMs") + + result = {"success": [], "failed": [], "skipped": []} + vm_ids = list(vm_imports.keys()) + + # Use job logger if available, otherwise standard logger + log = job.logger if job else logger + + for idx, vm_id in enumerate(vm_ids, start=1): + # Check for job cancellation every 5 VMs + if job and idx % 5 == 0: + job.job.refresh_from_db() + job_status = job.job.status + status_value = job_status.value if hasattr(job_status, "value") else job_status + if status_value in ("failed", "errored"): + log.warning(f"Job cancelled at VM {idx} of {len(vm_ids)}") + break + log.info(f"Imported VM {idx} of {len(vm_ids)}") + + try: + # Fetch device data (uses cache helper) + libre_device = fetch_device_with_cache(vm_id, api, api.server_key, libre_devices_cache) + + if not libre_device: + result["failed"].append( + { + "device_id": vm_id, + "error": f"Device {vm_id} not found in LibreNMS", + } + ) + log.error(f"Device {vm_id} not found in LibreNMS") + continue + + # Validate as VM + use_sysname_opt = sync_options.get("use_sysname", True) if sync_options else True + strip_domain_opt = sync_options.get("strip_domain", False) if sync_options else False + validation = validate_device_for_import( + libre_device, + import_as_vm=True, + api=api, + use_sysname=use_sysname_opt, + strip_domain=strip_domain_opt, + server_key=api.server_key, + ) + + # Check if VM already exists + if validation.get("existing_device"): + result["skipped"].append( + { + "device_id": vm_id, + "reason": f"VM already exists: {validation['existing_device'].name}", + } + ) + log.info(f"VM already exists: {validation['existing_device'].name}") + continue + + # Apply manual cluster and role selections + vm_mappings = vm_imports[vm_id] + cluster_id = vm_mappings.get("cluster_id") + role_id = vm_mappings.get("device_role_id") + + if cluster_id: + cluster = Cluster.objects.filter(id=cluster_id).first() + if cluster: + apply_cluster_to_validation(validation, cluster) + + role = None + if role_id: + role = DeviceRole.objects.filter(id=role_id).first() + if role: + apply_role_to_validation(validation, role, is_vm=True) + + # Determine VM name + use_sysname = sync_options.get("use_sysname", True) if sync_options else True + strip_domain = sync_options.get("strip_domain", False) if sync_options else False + + vm_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=strip_domain, + device_id=vm_id, + ) + + # Update validation with computed name + libre_device["_computed_name"] = vm_name + + # Create VM + vm = create_vm_from_librenms( + libre_device, validation, use_sysname=use_sysname, role=role, server_key=api.server_key + ) + + result["success"].append( + { + "device_id": vm_id, + "device": vm, + "message": f"VM {vm.name} created successfully", + } + ) + log.info(f"Successfully imported VM {vm.name} (ID: {vm_id})") + + except Exception as vm_error: + log.error(f"Failed to import VM {vm_id}: {vm_error}", exc_info=True) + result["failed"].append({"device_id": vm_id, "error": str(vm_error)}) + + return result diff --git a/netbox_librenms_plugin/jobs.py b/netbox_librenms_plugin/jobs.py index bb3d00e37d..f476c1f7e7 100644 --- a/netbox_librenms_plugin/jobs.py +++ b/netbox_librenms_plugin/jobs.py @@ -46,6 +46,8 @@ def run( show_disabled, exclude_existing=False, server_key=None, + use_sysname=True, + strip_domain=False, **kwargs, ): """ @@ -60,6 +62,8 @@ def run( show_disabled: Whether to include disabled devices exclude_existing: Whether to exclude devices that already exist in NetBox server_key: Optional LibreNMS server key for multi-server setups + use_sysname: If True, prefer sysName over hostname + strip_domain: If True, strip domain suffix from device name **kwargs: Additional job parameters """ from netbox_librenms_plugin.import_utils import process_device_filters @@ -88,6 +92,8 @@ def run( show_disabled=show_disabled, exclude_existing=exclude_existing, job=self, + use_sysname=use_sysname, + strip_domain=strip_domain, ) # Store device IDs for result retrieval @@ -107,6 +113,8 @@ def run( "filters": filters, "server_key": server_key, "vc_detection_enabled": vc_detection_enabled, + "use_sysname": use_sysname, + "strip_domain": strip_domain, "cache_timeout": api.cache_timeout, "cached_at": cached_at, "completed": True, diff --git a/netbox_librenms_plugin/librenms_api.py b/netbox_librenms_plugin/librenms_api.py index 5de9db6c25..3291aad0cc 100644 --- a/netbox_librenms_plugin/librenms_api.py +++ b/netbox_librenms_plugin/librenms_api.py @@ -190,7 +190,9 @@ def get_librenms_id(self, obj): If found via API, stores ID in custom field if available, otherwise caches the value. """ - librenms_id = obj.cf.get("librenms_id") + from netbox_librenms_plugin.utils import get_librenms_device_id + + librenms_id = get_librenms_device_id(obj, self.server_key) if librenms_id: return librenms_id @@ -254,7 +256,9 @@ def _store_librenms_id(self, obj, librenms_id): None """ if "librenms_id" in obj.cf: - obj.custom_field_data["librenms_id"] = librenms_id + from netbox_librenms_plugin.utils import set_librenms_device_id + + set_librenms_device_id(obj, librenms_id, self.server_key) obj.save() else: # Use cache as fallback @@ -686,6 +690,51 @@ def get_device_inventory(self, device_id): except requests.exceptions.RequestException as e: return False, str(e) + def get_device_transceivers(self, device_id): + """ + Fetch all transceiver data for a device from LibreNMS. + + Route: /api/v0/devices/{device_id}/transceivers + + This is a separate data source from entity inventory. Some vendors + (e.g., Nokia/SROS) don't expose SFPs via ENTITY-MIB but do report + them through vendor-specific MIBs which LibreNMS surfaces here. + + Args: + device_id: LibreNMS device ID + + Returns: + tuple: (success: bool, data: list) + + Example transceiver item: + { + "port_id": 519, + "entity_physical_index": 1610899520, + "type": "CFP2/QSFP28", + "model": "3HE10550AARA01", + "serial": "X42AU0D", + "channels": 4, + "connector": "LC", + "wavelength": 1301, + ... + } + """ + try: + response = requests.get( + f"{self.librenms_url}/api/v0/devices/{device_id}/transceivers", + headers=self.headers, + timeout=DEFAULT_API_TIMEOUT, + verify=self.verify_ssl, + ) + response.raise_for_status() + + if response.status_code == 200: + data = response.json() + return True, data.get("transceivers", []) + return False, [] + except requests.exceptions.RequestException as e: + return False, str(e) + def get_poller_groups(self): """ Fetch all poller groups from LibreNMS. diff --git a/netbox_librenms_plugin/migrations/0009_add_devicetypemapping.py b/netbox_librenms_plugin/migrations/0009_add_devicetypemapping.py new file mode 100644 index 0000000000..dcd8fc4fd5 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0009_add_devicetypemapping.py @@ -0,0 +1,45 @@ +# Generated by Django 5.2.10 on 2026-02-17 11:48 + +import django.db.models.deletion +import netbox.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("dcim", "0225_gfk_indexes"), + ("extras", "0134_owner"), + ("netbox_librenms_plugin", "0008_librenmssettings_import_defaults"), + ] + + operations = [ + migrations.CreateModel( + name="DeviceTypeMapping", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ("librenms_hardware", models.CharField(max_length=255, unique=True)), + ("description", models.TextField(blank=True)), + ( + "netbox_device_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="librenms_mappings", + to="dcim.devicetype", + ), + ), + ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), + ], + options={ + "ordering": ["librenms_hardware"], + }, + bases=(netbox.models.deletion.DeleteMixin, models.Model), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0010_add_moduletypemapping.py b/netbox_librenms_plugin/migrations/0010_add_moduletypemapping.py new file mode 100644 index 0000000000..796bbceafd --- /dev/null +++ b/netbox_librenms_plugin/migrations/0010_add_moduletypemapping.py @@ -0,0 +1,49 @@ +# Generated by Django 5.2.10 on 2026-02-17 12:23 + +import django.db.models.deletion +import netbox.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("dcim", "0225_gfk_indexes"), + ("extras", "0134_owner"), + ("netbox_librenms_plugin", "0009_add_devicetypemapping"), + ] + + operations = [ + migrations.AlterModelOptions( + name="interfacetypemapping", + options={"ordering": ["librenms_type", "librenms_speed"]}, + ), + migrations.CreateModel( + name="ModuleTypeMapping", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ("librenms_model", models.CharField(max_length=255, unique=True)), + ("description", models.TextField(blank=True)), + ( + "netbox_module_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="librenms_mappings", + to="dcim.moduletype", + ), + ), + ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), + ], + options={ + "ordering": ["librenms_model"], + }, + bases=(netbox.models.deletion.DeleteMixin, models.Model), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0011_modulebaymapping.py b/netbox_librenms_plugin/migrations/0011_modulebaymapping.py new file mode 100644 index 0000000000..5b3c2c3be0 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0011_modulebaymapping.py @@ -0,0 +1,38 @@ +# Generated by Django 5.2.10 on 2026-02-17 12:29 + +import netbox.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("extras", "0134_owner"), + ("netbox_librenms_plugin", "0010_add_moduletypemapping"), + ] + + operations = [ + migrations.CreateModel( + name="ModuleBayMapping", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ("librenms_name", models.CharField(max_length=255)), + ("librenms_class", models.CharField(blank=True, max_length=50)), + ("netbox_bay_name", models.CharField(max_length=255)), + ("description", models.TextField(blank=True)), + ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), + ], + options={ + "ordering": ["librenms_name"], + "unique_together": {("librenms_name", "librenms_class")}, + }, + bases=(netbox.models.deletion.DeleteMixin, models.Model), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0012_add_is_regex_to_modulebaymapping.py b/netbox_librenms_plugin/migrations/0012_add_is_regex_to_modulebaymapping.py new file mode 100644 index 0000000000..52ff053e20 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0012_add_is_regex_to_modulebaymapping.py @@ -0,0 +1,18 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("netbox_librenms_plugin", "0011_modulebaymapping"), + ] + + operations = [ + migrations.AddField( + model_name="modulebaymapping", + name="is_regex", + field=models.BooleanField( + default=False, + help_text="Treat LibreNMS Name as a regex pattern with backreferences in NetBox Bay Name", + ), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0013_normalizationrule.py b/netbox_librenms_plugin/migrations/0013_normalizationrule.py new file mode 100644 index 0000000000..71d1f80509 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0013_normalizationrule.py @@ -0,0 +1,93 @@ +"""Restore NormalizationRule model. + +The table was created by earlier migrations (0013 + 0014 in a previous branch) +and already exists in the database. This migration uses SeparateDatabaseAndState +so Django's ORM knows about the model without trying to CREATE the table again. +If the table doesn't exist (fresh install), the database_operations handle creation. +""" + +import django.db.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("dcim", "0001_initial"), + ("extras", "0001_initial"), + ("netbox_librenms_plugin", "0012_add_is_regex_to_modulebaymapping"), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.CreateModel( + name="NormalizationRule", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ( + "scope", + models.CharField( + choices=[ + ("module_type", "Module Type"), + ("device_type", "Device Type"), + ("module_bay", "Module Bay"), + ], + max_length=50, + ), + ), + ("match_pattern", models.CharField(max_length=500)), + ("replacement", models.CharField(max_length=500)), + ("priority", models.PositiveIntegerField(default=100)), + ("description", models.TextField(blank=True)), + ( + "manufacturer", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="normalization_rules", + to="dcim.manufacturer", + ), + ), + ( + "tags", + taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag"), + ), + ], + options={ + "ordering": ["scope", "priority", "pk"], + }, + ), + ], + database_operations=[ + migrations.RunSQL( + sql=""" + CREATE TABLE IF NOT EXISTS "netbox_librenms_plugin_normalizationrule" ( + "id" bigserial NOT NULL PRIMARY KEY, + "created" timestamp with time zone NULL, + "last_updated" timestamp with time zone NULL, + "custom_field_data" jsonb NOT NULL DEFAULT '{}'::jsonb, + "scope" varchar(50) NOT NULL, + "match_pattern" varchar(500) NOT NULL, + "replacement" varchar(500) NOT NULL, + "priority" integer NOT NULL DEFAULT 100 CHECK ("priority" >= 0), + "description" text NOT NULL DEFAULT '', + "manufacturer_id" bigint NULL REFERENCES "dcim_manufacturer" ("id") + DEFERRABLE INITIALLY DEFERRED + ); + CREATE INDEX IF NOT EXISTS "netbox_librenms_plugin_norm_mfg_idx" + ON "netbox_librenms_plugin_normalizationrule" ("manufacturer_id"); + """, + reverse_sql="DROP TABLE IF EXISTS netbox_librenms_plugin_normalizationrule;", + ), + ], + ), + ] diff --git a/netbox_librenms_plugin/models.py b/netbox_librenms_plugin/models.py index cd79f47550..3978d9c5f4 100644 --- a/netbox_librenms_plugin/models.py +++ b/netbox_librenms_plugin/models.py @@ -1,4 +1,8 @@ +import re + from dcim.choices import InterfaceTypeChoices +from dcim.models import DeviceType, Manufacturer, ModuleType +from django.core.exceptions import ValidationError from django.db import models from django.urls import reverse from netbox.models import NetBoxModel @@ -71,6 +75,205 @@ class Meta: """Meta options for InterfaceTypeMapping.""" unique_together = ["librenms_type", "librenms_speed"] + ordering = ["librenms_type", "librenms_speed"] def __str__(self): return f"{self.librenms_type} + {self.librenms_speed} -> {self.netbox_type}" + + +class DeviceTypeMapping(NetBoxModel): + """Map LibreNMS hardware strings to NetBox DeviceType objects.""" + + librenms_hardware = models.CharField( + max_length=255, + unique=True, + help_text="Hardware string as reported by LibreNMS (e.g., 'Juniper MX480 Internet Backbone Router')", + ) + netbox_device_type = models.ForeignKey( + DeviceType, + on_delete=models.CASCADE, + related_name="librenms_mappings", + help_text="The NetBox DeviceType this hardware string maps to", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this mapping", + ) + + def get_absolute_url(self): + """Return the URL for this mapping's detail page.""" + return reverse("plugins:netbox_librenms_plugin:devicetypemapping_detail", args=[self.pk]) + + class Meta: + """Meta options for DeviceTypeMapping.""" + + ordering = ["librenms_hardware"] + + def __str__(self): + return f"{self.librenms_hardware} -> {self.netbox_device_type}" + + +class ModuleTypeMapping(NetBoxModel): + """Map LibreNMS inventory model names to NetBox ModuleType objects.""" + + librenms_model = models.CharField( + max_length=255, + unique=True, + help_text="Model name from LibreNMS inventory (entPhysicalModelName)", + ) + netbox_module_type = models.ForeignKey( + ModuleType, + on_delete=models.CASCADE, + related_name="librenms_mappings", + help_text="The NetBox ModuleType this model name maps to", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this mapping", + ) + + def get_absolute_url(self): + """Return the URL for this mapping's detail page.""" + return reverse("plugins:netbox_librenms_plugin:moduletypemapping_detail", args=[self.pk]) + + class Meta: + """Meta options for ModuleTypeMapping.""" + + ordering = ["librenms_model"] + + def __str__(self): + return f"{self.librenms_model} -> {self.netbox_module_type}" + + +class ModuleBayMapping(NetBoxModel): + """Map LibreNMS inventory names to NetBox module bay names. + + Used when LibreNMS inventory names don't match NetBox bay names exactly. + For example: LibreNMS "Power Supply 1" β†’ NetBox "PS1". + When is_regex is True, librenms_name is treated as a regex pattern and + netbox_bay_name can use backreferences (\\1, \\2, etc.). + Mappings are global (not scoped to device type or manufacturer). + """ + + librenms_name = models.CharField( + max_length=255, + help_text="Name from LibreNMS inventory (entPhysicalName). " + "When 'Use Regex' is enabled, this is a Python regex pattern.", + ) + librenms_class = models.CharField( + max_length=50, + blank=True, + help_text="Optional entPhysicalClass filter (e.g. 'powerSupply', 'fan', 'module')", + ) + netbox_bay_name = models.CharField( + max_length=255, + help_text="NetBox module bay name to match. With regex, supports backreferences (\\1, \\2, etc.).", + ) + is_regex = models.BooleanField( + default=False, + help_text="Treat LibreNMS Name as a regex pattern with backreferences in NetBox Bay Name", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this mapping", + ) + + def clean(self): + """Validate that regex patterns compile when is_regex is True.""" + super().clean() + if self.is_regex: + try: + re.compile(self.librenms_name) + except re.error as e: + raise ValidationError({"librenms_name": f"Invalid regex: {e}"}) + + def get_absolute_url(self): + """Return the URL for this mapping's detail page.""" + return reverse("plugins:netbox_librenms_plugin:modulebaymapping_detail", args=[self.pk]) + + class Meta: + """Meta options for ModuleBayMapping.""" + + unique_together = ["librenms_name", "librenms_class"] + ordering = ["librenms_name"] + + def __str__(self): + cls = f" [{self.librenms_class}]" if self.librenms_class else "" + return f"{self.librenms_name}{cls} -> {self.netbox_bay_name}" + + +class NormalizationRule(NetBoxModel): + """Regex-based string normalization applied before matching lookups. + + Generic building block: a single rule engine handles normalization + for module types, device types, module bays, and future scopes. + Rules are applied in priority order; each transforms the string + for the next rule in the chain. + + Example – strip Nokia revision suffixes: + scope: module_type + match_pattern: ^(3HE\\w{5}[A-Z]{2})[A-Z]{2}\\d{2}$ + replacement: \\1 + Result: 3HE16474AARA01 β†’ 3HE16474AA + """ + + SCOPE_MODULE_TYPE = "module_type" + SCOPE_DEVICE_TYPE = "device_type" + SCOPE_MODULE_BAY = "module_bay" + + SCOPE_CHOICES = [ + (SCOPE_MODULE_TYPE, "Module Type"), + (SCOPE_DEVICE_TYPE, "Device Type"), + (SCOPE_MODULE_BAY, "Module Bay"), + ] + + scope = models.CharField( + max_length=50, + choices=SCOPE_CHOICES, + help_text="Which matching lookup this rule applies to", + ) + manufacturer = models.ForeignKey( + Manufacturer, + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="normalization_rules", + help_text="Optional: only apply this rule to items from this manufacturer. " + "Leave blank for vendor-agnostic rules.", + ) + match_pattern = models.CharField( + max_length=500, + help_text="Regex pattern to match against input string (Python re syntax)", + ) + replacement = models.CharField( + max_length=500, + help_text="Replacement string (supports regex back-references \\1, \\2, …)", + ) + priority = models.PositiveIntegerField( + default=100, + help_text="Lower values run first. Rules chain: each transforms the output of the previous.", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this rule", + ) + + def clean(self): + """Validate that match_pattern compiles as a regex.""" + super().clean() + try: + re.compile(self.match_pattern) + except re.error as e: + raise ValidationError({"match_pattern": f"Invalid regex: {e}"}) + + def get_absolute_url(self): + """Return the URL for this rule's detail page.""" + return reverse("plugins:netbox_librenms_plugin:normalizationrule_detail", args=[self.pk]) + + class Meta: + """Meta options for NormalizationRule.""" + + ordering = ["scope", "priority", "pk"] + + def __str__(self): + return f"[{self.get_scope_display()}] {self.match_pattern} β†’ {self.replacement}" diff --git a/netbox_librenms_plugin/navigation.py b/netbox_librenms_plugin/navigation.py index a08e62740f..052c06363c 100644 --- a/netbox_librenms_plugin/navigation.py +++ b/netbox_librenms_plugin/navigation.py @@ -31,6 +31,74 @@ ), ), ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:devicetypemapping_list", + link_text="Device Type Mappings", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:devicetypemapping_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:devicetypemapping_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:moduletypemapping_list", + link_text="Module Type Mappings", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:moduletypemapping_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:moduletypemapping_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:modulebaymapping_list", + link_text="Module Bay Mappings", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:modulebaymapping_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:modulebaymapping_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:normalizationrule_list", + link_text="Normalization Rules", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:normalizationrule_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:normalizationrule_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), ), ), ( diff --git a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js index c89473c02a..2e130b1284 100644 --- a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js +++ b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js @@ -310,6 +310,8 @@ * * @param {HTMLElement} modalElement - The modal element to hide * @param {Object} fallbackBackdropRef - Reference object containing fallback backdrop (deprecated) + * WONTFIX: fallbackBackdropRef is unused β€” _hideManual uses querySelector which is + * correct for this plugin since only one modal is ever open at a time (Tabler, no Bootstrap). */ function hideModal(modalElement, fallbackBackdropRef) { if (!modalElement) { @@ -317,6 +319,14 @@ } const manager = new ModalManager(modalElement); + + // Try to recover an existing Bootstrap instance before falling back to manual + if (typeof bootstrap !== 'undefined' && bootstrap.Modal) { + manager.instance = bootstrap.Modal.getInstance(modalElement); + } else if (typeof window.bootstrap !== 'undefined' && window.bootstrap.Modal) { + manager.instance = window.bootstrap.Modal.getInstance(modalElement); + } + manager.hide(); } @@ -338,6 +348,7 @@ function pollJobStatus(jobId, jobPk, pollUrl, baseUrl, originalFilters, deviceCount) { const messageEl = document.getElementById('filter-progress-message'); const cancelBtn = document.getElementById('cancel-filter-btn'); + const filterModal = document.getElementById('filter-processing-modal'); // Get CSRF token from cookie or form (needed for cancel and status sync) let csrfToken = getCookie('csrftoken'); @@ -402,11 +413,8 @@ messageEl.textContent = 'Job already completed, loading results...'; } cancelBtn.textContent = 'Completed'; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => { @@ -448,11 +456,8 @@ messageEl.textContent = 'Job cancelled successfully.'; } cancelBtn.textContent = 'Cancelled'; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => { @@ -469,11 +474,8 @@ messageEl.textContent = 'Job stopped (status sync failed).'; } cancelBtn.textContent = 'Stopped'; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => { @@ -486,11 +488,8 @@ messageEl.textContent = 'Job completed, loading results...'; } cancelBtn.textContent = 'Completed'; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => { @@ -505,11 +504,8 @@ } cancelBtn.textContent = 'Close'; cancelBtn.disabled = false; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => window.location.href = baseUrl, 1000); @@ -590,11 +586,8 @@ if (statusValue === 'completed' || statusValue === 'finished') { pollingStopped = true; // Stop future polls - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } // Small delay to let modal close before redirect @@ -604,21 +597,15 @@ return; // Stop polling } else if (statusValue === 'stopped') { pollingStopped = true; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => window.location.href = baseUrl, 100); } else if (statusValue === 'failed') { pollingStopped = true; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } const errorMsg = data.data?.error; @@ -628,11 +615,8 @@ setTimeout(() => window.location.href = baseUrl, 100); } else if (statusValue === 'errored') { pollingStopped = true; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } const errorMsg = data.data?.error || 'Job encountered an error. Please try again.'; @@ -1031,11 +1015,8 @@ if (failedCount && failedCount.dataset.failedCount === '0') { setTimeout(() => { const resultsModal = document.getElementById('import-results-modal'); - if (resultsModal && typeof bootstrap !== 'undefined' && bootstrap.Modal) { - const modalInstance = bootstrap.Modal.getInstance(resultsModal); - if (modalInstance) { - modalInstance.hide(); - } + if (resultsModal) { + hideModal(resultsModal); } window.location.reload(); }, MODAL_AUTO_CLOSE_MS); @@ -1113,6 +1094,12 @@ } showModal(modalElement, fallbackBackdropRef); + + // Re-initialize tooltips for newly swapped modal content + if (typeof bootstrap !== 'undefined' && bootstrap.Tooltip) { + const tooltipEls = modalContent.querySelectorAll('[data-bs-toggle="tooltip"]'); + [...tooltipEls].map(el => new bootstrap.Tooltip(el)); + } } document.body.addEventListener('htmx:afterSwap', ensureModalVisible); diff --git a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js index cd470af1b1..0b42112b24 100644 --- a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js +++ b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js @@ -153,11 +153,15 @@ function initializeCountdowns() { if (window.vlanCountdownInterval) { clearInterval(window.vlanCountdownInterval); } + if (window.moduleCountdownInterval) { + clearInterval(window.moduleCountdownInterval); + } window.interfaceCountdownInterval = initializeCountdown("countdown-timer"); window.cableCountdownInterval = initializeCountdown("cable-countdown-timer"); window.ipCountdownInterval = initializeCountdown("ip-countdown-timer"); window.vlanCountdownInterval = initializeCountdown("vlan-countdown-timer"); + window.moduleCountdownInterval = initializeCountdown("module-countdown-timer"); } // ============================================ diff --git a/netbox_librenms_plugin/tables/device_status.py b/netbox_librenms_plugin/tables/device_status.py index 8bf978ccd4..3f03764cef 100644 --- a/netbox_librenms_plugin/tables/device_status.py +++ b/netbox_librenms_plugin/tables/device_status.py @@ -444,7 +444,7 @@ def render_actions(self, value, record): buttons = [] if existing: - # Link to existing device/VM in NetBox + # Link to existing device/VM in NetBox + details button for conflict resolution if isinstance(existing, VirtualMachine): url_name = "virtualization:virtualmachine" title = "View VM in NetBox" @@ -457,6 +457,50 @@ def render_actions(self, value, record): f'' ) + + # Add details/conflict button for conflict resolution actions + details_url = self._build_validation_details_url(device_id, validation) + match_type = validation.get("existing_match_type", "") + serial_action = validation.get("serial_action") + has_mismatch = validation.get("device_type_mismatch", False) + has_actions = match_type == "hostname" or (match_type == "serial" and serial_action is not None) + has_name_sync = validation.get("name_sync_available", False) + has_sync_needed = match_type == "librenms_id" and serial_action in ("update_serial", "conflict") + + if has_mismatch: + btn_class = "btn-outline-danger" + btn_icon = "mdi-alert-circle" + btn_label = " Conflict" + elif has_actions: + btn_class = "btn-outline-warning" + btn_icon = "mdi-alert" + btn_label = " Conflict" + elif has_name_sync or has_sync_needed: + btn_class = "btn-outline-warning" + btn_icon = "mdi-information-outline" + btn_label = " Details" + elif match_type == "librenms_id" and validation.get("librenms_id_needs_migration"): + btn_class = "btn-outline-warning" + btn_icon = "mdi-database-alert" + btn_label = " Legacy ID" + else: + btn_class = "btn-outline-success" + btn_icon = "mdi-check-circle" + btn_label = "" + + btn_title = "Resolve conflict" if (has_actions or has_mismatch) else "View details" + aria_attr = f'aria-label="{btn_title}" ' if btn_label == "" else "" + buttons.append( + f'' + ) elif is_ready: # Ready to import - show Import and Details buttons details_url = self._build_validation_details_url(device_id, validation) @@ -473,7 +517,7 @@ def render_actions(self, value, record): f'", + url, + self.csrf_token, + record.get("module_bay_id", ""), + record.get("module_type_id", ""), + record.get("serial", ""), + ) + ) + + # Install branch button for parents with installable children + if record.get("has_installable_children") and record.get("ent_physical_index"): + url = reverse("plugins:netbox_librenms_plugin:install_branch", kwargs={"pk": self.device.pk}) + buttons.append( + format_html( + '
' + '' + '' + '
", + url, + self.csrf_token, + record.get("ent_physical_index", ""), + ) + ) + + return format_html("{}", format_html("".join(str(b) for b in buttons))) if buttons else "" diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html new file mode 100644 index 0000000000..ce6e430bfa --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html @@ -0,0 +1,31 @@ +{% load helpers %} +{% include 'inc/messages.html' %} + + +{% if module_sync.table %} +
+
+ + Showing inventory items from LibreNMS matched against NetBox module bays and module types. + +
+ {% if module_sync.cache_expiry %} +
+ Cache expires in: +
+ {% endif %} +
+ +
+ {% include 'netbox_librenms_plugin/inc/paginator.html' with table=module_sync.table %} + {% include 'inc/table.html' with table=module_sync.table %} + {% include 'netbox_librenms_plugin/inc/paginator.html' with table=module_sync.table %} +
+{% else %} +
+
+ +

No inventory data loaded. Click Refresh Modules to fetch data from LibreNMS.

+
+
+{% endif %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping.html new file mode 100644 index 0000000000..3894441179 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping.html @@ -0,0 +1,28 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} + +{% block content %} +
+
+
+ + + + + + + + + + + + + + + +
LibreNMS HardwareNetBox Device TypeDescription
{{ object.librenms_hardware }}{{ object.netbox_device_type }}{{ object.description|default:"β€”" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping_list.html new file mode 100644 index 0000000000..06c95270b3 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping_list.html @@ -0,0 +1,12 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Device Type Mapping

+

Map LibreNMS hardware strings to NetBox device types. + When importing devices from LibreNMS, these mappings are checked first before + falling back to exact part number / model matching.

+

Example: Map "Juniper MX480 Internet Backbone Router" to device type "MX480"

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html index 31ea3df6cb..9b16c47436 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html @@ -1,325 +1,586 @@ {# HTMX template for device validation details modal #} -{# Shows detailed reasons why a device cannot be imported #} +{# Redesigned to match the sync page's clean table layout #}
- Import Validation: {{ libre_device.hostname }} + {% if validation.existing_device %} + {% if validation.device_type_mismatch %} + + {% elif validation.existing_match_type == 'librenms_id' %} + + {% else %} + + {% endif %} + {% elif validation.is_ready %} + + {% elif validation.can_import %} + + {% else %} + + {% endif %} + Import Validation: {{ validation.resolved_name|default:libre_device.sysName|default:libre_device.hostname }}
- {# Device Summary #} -
-
- LibreNMS Device Information + {# Compute existing device URL once for use throughout #} + {% if validation.existing_device %} + {% if validation.import_as_vm or validation.existing_device.cluster %} + {% url 'virtualization:virtualmachine' pk=validation.existing_device.pk as existing_device_url %} + {% else %} + {% url 'dcim:device' pk=validation.existing_device.pk as existing_device_url %} + {% endif %} + {% endif %} + + {# Row: LibreNMS Status (left) + Device Info table (right) #} +
+ + {# Left: LibreNMS Status card #} +
+
+
LibreNMS Status
+
+ + + + + + + + + + + + + + + + + + + + + + + +
Status + {% if libre_device.status == 1 or libre_device.status == "1" %} + Up + {% elif libre_device.status == 0 or libre_device.status == "0" %} + Down + {% else %} + Unknown + {% endif %} +
Hostname{{ libre_device.hostname }}
ID{{ libre_device.device_id }}
IP{{ libre_device.ip|default:"β€”" }}
Location{{ libre_device.location|default:"β€”" }}
+
+
-
-
-
Hostname:
-
{{ libre_device.hostname }}
-
System Name:
-
{{ libre_device.sysName|default:"N/A" }}
+ {# Right: Device Information table #} +
+
+
+ Device Information + + {% if use_sysname %} + sysName + {% else %} + hostname + {% endif %} + {% if strip_domain %} + Domain stripped + {% endif %} + +
+
+ + + + + + + + + + {# Name row #} + + + + + -
LibreNMS ID:
-
{{ libre_device.device_id }}
+ {% if not validation.import_as_vm %} + {# Site row #} + + + + + -
IP Address:
-
{{ libre_device.ip|default:"N/A" }}
+ {# Device Type row #} + + + + + + {% endif %} -
Hardware:
-
{{ libre_device.hardware|default:"N/A" }}
+ {# Serial row (only for devices) #} + {% if not validation.import_as_vm %} + + + + + + {% endif %} -
OS:
-
{{ libre_device.os|default:"N/A" }}
+ {# Role row #} + + + + + -
Location:
-
{{ libre_device.location|default:"N/A" }}
+ {# Platform row #} + + + + + -
Status:
-
- {% if libre_device.status == 1 %} - Up - {% elif libre_device.status == 0 %} - Down - {% else %} - Unknown - {% endif %} -
- + {% if validation.import_as_vm %} + {# Cluster row (VMs only) #} + + + + + + {% else %} + {# Rack row #} + + + + + + {% endif %} + + {# Primary IP row #} + + + + + + +
FieldNetBox ValueLibreNMS Value
Name + {% if validation.existing_device %} + + {{ validation.existing_device.name }} + + {% if validation.name_sync_available %} +
+ {% csrf_token %} + + + +
+ {% endif %} + {% else %} + New device + {% endif %} + 		{{ libre_device.sysName|default:libre_device.hostname }}
Site + {% if validation.existing_device and validation.existing_device.site %} + {{ validation.existing_device.site }} + {% if validation.site.site and validation.existing_device.site.pk == validation.site.site.pk %} + + {% endif %} + {% elif validation.site.site %} + {{ validation.site.site.name }} + + {% else %} + No matching site + {% endif %} + {{ libre_device.location|default:"β€”" }}
Device Type + {% if validation.device_type_mismatch %} + + {{ validation.existing_device.device_type }} + + +
+ {% csrf_token %} + + + + +
+ {% elif validation.existing_device and validation.existing_device.device_type %} + {{ validation.existing_device.device_type }} + {% if sync_info and not sync_info.device_type_synced and sync_info.librenms_device_type %} +
+ {% csrf_token %} + + + +
+ {% elif validation.device_type.device_type %} + + {% endif %} + {% elif validation.device_type.device_type %} + + {{ validation.device_type.device_type }} + + + {% else %} + No matching type + {% endif %} + 		{{ libre_device.hardware|default:"β€”" }}
Serial + {% if validation.existing_device %} + {% if validation.existing_device.serial %} + {{ validation.existing_device.serial }} + {% else %} + Not set + {% endif %} + {% if sync_info and not sync_info.serial_synced %} + {% if validation.serial_action == 'conflict' %} + + + + {% else %} +
+ {% csrf_token %} + + + +
+ {% endif %} + {% elif sync_info and sync_info.serial_synced and sync_info.librenms_serial != '-' %} + + {% endif %} + {% else %} + β€” + {% endif %} + 		+ {{ libre_device.serial|default:"β€”" }} + {% if validation.serial_duplicate %} + Conflict + {% endif %} +
+ {% if validation.import_as_vm %}Role{% else %}Device Role{% endif %} + + {% if validation.existing_device and validation.existing_device.role %} + {{ validation.existing_device.role }} + + {% elif validation.device_role.role %} + {{ validation.device_role.role.name }} + + {% else %} + No role assigned + {% endif %} + β€”
Platform + {% if validation.existing_device and validation.existing_device.platform %} + {{ validation.existing_device.platform }} + {% if sync_info and not sync_info.platform_synced %} + {% if sync_info.platform_info.platform_exists %} + {% if not validation.import_as_vm and not validation.existing_device.cluster %} +
+ {% csrf_token %} + + + +
+ {% endif %} + {% else %} + + + + {% endif %} + {% elif sync_info and sync_info.platform_synced %} + + {% endif %} + {% elif validation.platform.platform %} + {{ validation.platform.platform.name }} + + {% elif sync_info and sync_info.platform_info.platform_exists %} + Not set + {% if validation.existing_device and not validation.import_as_vm and not validation.existing_device.cluster %} +
+ {% csrf_token %} + + + +
+ {% endif %} + {% else %} + Optional + {% endif %} + 		{{ libre_device.os|default:"β€”" }}
Cluster + {% if validation.cluster.cluster %} + {{ validation.cluster.cluster.name }} + + {% else %} + No cluster assigned + {% endif %} + β€”
Rack + {% if validation.rack.rack %} + {% if validation.rack.rack.location %} + {{ validation.rack.rack.location.name }} β€” {{ validation.rack.rack.name }} + {% else %} + {{ validation.rack.rack.name }} + {% endif %} + + {% else %} + Optional + {% endif %} + β€”
Primary IP + {% if validation.existing_device and validation.existing_device.primary_ip %} + {{ validation.existing_device.primary_ip }} + + {% elif libre_device.ip %} + {{ libre_device.ip }} + + {% else %} + No primary IP + {% endif %} + {{ libre_device.ip|default:"β€”" }}
+
+
- {# Validation Status #} -
-
- {% if validation.existing_device %} - {% if validation.existing_match_type == 'librenms_id' %} - {# Device was imported via this plugin #} -
-
- Device Already Imported -
-

- This device has already been imported to NetBox as - - {{ validation.existing_device.name }} - -

-

- - Matched by LibreNMS ID ({{ libre_device.device_id }}) - device is linked and synchronized. -

-
- {% elif validation.existing_match_type == 'hostname' %} - {# Device exists with same hostname but wasn't imported from LibreNMS #} -
-
- Device Exists (Hostname Match) -
-

- A device with the same hostname exists in NetBox as - - {{ validation.existing_device.name }} - -

-

- - Matched by hostname only - this device is not linked to LibreNMS. - If this is the same device, consider adding the LibreNMS ID custom field manually. -

-
- {% elif validation.existing_match_type == 'primary_ip' %} - {# Device exists with same IP but wasn't imported from LibreNMS #} -
-
- Device Exists (IP Match) -
-

- A device with the same IP address ({{ libre_device.ip }}) exists in NetBox as - - {{ validation.existing_device.name }} - -

-

- - Matched by IP address only - verify this is the same device. - If correct, consider adding the LibreNMS ID custom field manually. -

-
+ {# Status & Actions #} + {% if validation.existing_device %} + {% if validation.existing_match_type == 'librenms_id' %} +
+ {% if existing_id_servers %} + {% for srv in existing_id_servers %} + Linked β€” ID {{ srv.device_id }} @ {{ srv.display_name }} + {% endfor %} {% else %} - {# Fallback for unknown match type #} -
-
- Device Already Exists -
-

- This device already exists in NetBox as - - {{ validation.existing_device.name }} - -

-

- No import is needed. -

-
+ Linked β€” ID {{ libre_device.device_id }} {% endif %} - {% elif validation.is_ready %} -
-
- Ready to Import -
-

- All prerequisites are met. This device can be imported into NetBox without any issues. -

+ {% if validation.name_matches %} + Name match + {% elif validation.name_sync_available %} + Name differs + {% endif %} + {% if validation.serial_confirmed %} + Serial confirmed + {% elif validation.serial_action == 'conflict' %} + Serial conflict + {% elif validation.serial_action == 'update_serial' %} + Serial differs + {% endif %} + {% if validation.device_type_mismatch %} + Type mismatch + {% endif %} + {% if validation.librenms_id_needs_migration %} + Legacy ID format + {% endif %} +
+ {% if validation.librenms_id_needs_migration %} +
+
+ {% csrf_token %} + + + {% if not validation.serial_confirmed %} +
+ + +
+ {% endif %} + +
+
+ {% endif %} + + {% elif validation.existing_match_type == 'hostname' %} +
+ Hostname match + {% if validation.serial_confirmed %} + Serial confirmed + {% elif validation.serial_action == 'conflict' %} + Serial conflict + {% elif validation.serial_action == 'update_serial' %} + Serial differs + {% endif %} + {% if validation.device_type_mismatch %} + Type mismatch + {% endif %} + + β€” Exists as + {{ validation.existing_device.name }}, + not linked to LibreNMS. + +
+ {% if validation.serial_action == 'conflict' %} +
+ + Import blocked: The incoming serial number is already assigned to another device in NetBox. + Resolve the duplicate serial before linking.
- {% elif validation.can_import %} -
-
- Can Import with Warnings -
-

- This device can be imported, but there are some warnings to review below. -

+ {% elif validation.import_as_vm %} +
+ + Hostname match found for a VM β€” use the import action to proceed.
{% else %} -
-
- Cannot Import -
-

- This device cannot be imported due to validation issues listed below. -

-
- {% endif %} -
-
- - {# Validation Details #} -
-
- NetBox Import Matching -
-
-

- - The following NetBox objects will be used when importing this - {% if validation.import_as_vm %}Virtual Machine{% else %}Device{% endif %}: - -

- - - {% if validation.import_as_vm %} - {# VM-specific fields #} - - - - - - - - - - - - - - - +
+
+ {% csrf_token %} + + {% if validation.device_type_mismatch %} +
+ + +
+ {% endif %} + {% if validation.serial_action == 'update_serial' %} + + {% else %} - {# Device-specific fields #} -
- - - - - - - - - - - - - + + + {% endif %} + + + {% endif %} - - - - + {% elif validation.existing_match_type == 'serial' %} +
+ Serial match + {% if validation.serial_action == 'hostname_differs' %} + Name differs + {% elif validation.serial_action == 'link' %} + Name match + {% endif %} + {% if validation.device_type_mismatch %} + Type mismatch + {% endif %} + + β€” Exists as + {{ validation.existing_device.name }}, + not linked to LibreNMS. + +
+
+
+ {% csrf_token %} + + {% if validation.device_type_mismatch %} +
+ + +
+ {% endif %} + {% if validation.serial_action == 'link' %} + + + {% elif validation.serial_action == 'hostname_differs' %} + {% endif %} + +
- {# Common fields for both Device and VM #} - - - - + {% elif validation.existing_match_type == 'primary_ip' %} +
+ IP match + + β€” Device with IP {{ libre_device.ip }} exists as + {{ validation.existing_device.name }}. + Consider adding LibreNMS ID manually. + +
- {% if not validation.import_as_vm %} - - - - - {% endif %} + {% else %} +
+ + Exists β€” Device already exists as + {{ validation.existing_device.name }}. +
+ {% endif %} - - - - - -
Import Type: - - Virtual Machine -
Cluster: - {% if validation.cluster.cluster %} - - {{ validation.cluster.cluster.name }} - {% else %} - - No cluster assigned (required for VMs) - {% endif %} -
Role: - {% if validation.device_role.role %} - - {{ validation.device_role.role.name }} - {% else %} - - Optional - not assigned - {% endif %} -
Import Type: - - Device -
Site: - {% if validation.site.site %} - - {{ validation.site.site.name }} - {% else %} - - No matching site found - {% endif %} -
Device Type: - {% if validation.device_type.device_type %} - - {{ validation.device_type.device_type }} - {% else %} - - No matching device type found - {% endif %} -
Device Role: - {% if validation.device_role.role %} - - {{ validation.device_role.role.name }} - {% else %} - - No device role assigned - {% endif %} -
Platform: - {% if validation.platform.platform %} - - {{ validation.platform.platform.name }} - {% else %} - - Optional - will be auto-mapped if available - {% endif %} -
Rack: - {% if validation.rack.rack %} - - - {% if validation.rack.rack.location %} - {{ validation.rack.rack.location.name }} - {{ validation.rack.rack.name }} - {% else %} - {{ validation.rack.rack.name }} - {% endif %} - - {% else %} - - Optional - not assigned - {% endif %} -
Primary IP: - {% if libre_device.ip %} - - {{ libre_device.ip }} - {% else %} - - No primary IP - will import without - {% endif %} -
+ {% elif validation.is_ready %} +
+ + Ready to Import β€” All prerequisites are met.
-
+ {% elif validation.can_import %} +
+ + Can Import β€” Review warnings below before importing. +
+ {% else %} +
+ + Cannot Import β€” Validation issues prevent import. +
+ {% endif %} {# Warnings #} {% if validation.warnings %}
-
+
Warnings ({{ validation.warnings|length }})
-
-

These warnings won't prevent import but should be reviewed:

-
    +
    +
      {% for warning in validation.warnings %} -
    • {{ warning }}
      • +
    • {{ warning }}
      • {% endfor %}
    @@ -331,23 +592,24 @@
    {% if validation.existing_device %} {% if validation.import_as_vm or validation.existing_device.cluster %} - {# It's a VM #} - + View VM in NetBox {% else %} - {# It's a Device #} - - View Device in NetBox + + View in NetBox + {% if validation.existing_match_type == 'librenms_id' %} + + Full Sync Page + + {% endif %} {% endif %} {% endif %} - -
    diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/inc/_module_sync.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/inc/_module_sync.html new file mode 100644 index 0000000000..4c2c5ae65d --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/inc/_module_sync.html @@ -0,0 +1,27 @@ +{% load helpers %} + + +
    +

    Module Sync

    +
    +
    + {% csrf_token %} + {% if has_librenms_id %} + {% with model_name=object|meta:"model_name" %} + {% if model_name == "device" %} + + {% endif %} + {% endwith %} + {% endif %} +
    +
    +
    + + +
    + {% include 'netbox_librenms_plugin/_module_sync_content.html' %} +
    diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_import.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_import.html index 0013ec5be3..f5c0eeaf49 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_import.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_import.html @@ -399,7 +399,7 @@
    {# HTMX Modal for import actions #}
Device Information Sync {% include 'netbox_librenms_plugin/_ipaddress_sync.html' %}
+ {% if module_sync %} +
+ {% include 'netbox_librenms_plugin/inc/_module_sync.html' %} +
+ {% endif %} + {% with model_name=object|meta:"model_name" %} {% if model_name == "device" %}
+
+
+ + + + + + + + + + + + + + + + + +
LibreNMS NameLibreNMS ClassNetBox Bay NameDescription
{{ object.librenms_name }}{{ object.librenms_class|default:"β€”" }}{{ object.netbox_bay_name }}{{ object.description|default:"β€”" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping_list.html new file mode 100644 index 0000000000..fb87f901ec --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping_list.html @@ -0,0 +1,12 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Module Bay Mapping

+

Map LibreNMS inventory container names to NetBox module bay names. + When synchronizing modules from LibreNMS, these mappings determine which + NetBox module bay a LibreNMS component should be installed into.

+

Example: Map "Linecard(slot 1)" to "Slot 1", or "Power Supply 1" to "PS1"

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping.html new file mode 100644 index 0000000000..019b0e51ed --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping.html @@ -0,0 +1,28 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} + +{% block content %} +
+
+
+ + + + + + + + + + + + + + + +
LibreNMS ModelNetBox Module TypeDescription
{{ object.librenms_model }}{{ object.netbox_module_type }}{{ object.description|default:"β€”" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping_list.html new file mode 100644 index 0000000000..4cfc22d592 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping_list.html @@ -0,0 +1,12 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Module Type Mapping

+

Map LibreNMS inventory model names (entPhysicalModelName) to NetBox module types. + When synchronizing modules from LibreNMS, these mappings are checked first before + falling back to exact model / part number matching.

+

Example: Map "710-017414" to module type "WS-X4908-10GE"

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule.html new file mode 100644 index 0000000000..a1be7537a3 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule.html @@ -0,0 +1,34 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} + +{% block content %} +
+
+
+ + + + + + + + + + + + + + + + + + + + + +
ScopeManufacturerMatch PatternReplacementPriorityDescription
{{ object.get_scope_display }}{% if object.manufacturer %}{{ object.manufacturer }}{% else %}β€”{% endif %}{{ object.match_pattern }}{{ object.replacement }}{{ object.priority }}{{ object.description|default:"β€”" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule_list.html new file mode 100644 index 0000000000..d543141680 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule_list.html @@ -0,0 +1,16 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Normalization Rules

+

Regex-based string normalization applied before matching lookups. + When a LibreNMS string doesn't match any NetBox object or mapping entry, + normalization rules transform it (e.g. strip revision suffixes) and retry.

+

Rules are chained in priority order per scope. One rule engine serves + module types, device types, and module bays.

+

Example β€” strip Nokia revision suffixes:
+ ^(3HE\w{5}[A-Z]{2})[A-Z]{2}\d{2}$ β†’ \1
+ Turns 3HE16474AARA01 into 3HE16474AA which matches the part number.

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/tests/test_background_jobs.py b/netbox_librenms_plugin/tests/test_background_jobs.py index 13dca842d8..be7f7af377 100644 --- a/netbox_librenms_plugin/tests/test_background_jobs.py +++ b/netbox_librenms_plugin/tests/test_background_jobs.py @@ -686,6 +686,8 @@ def test_load_success_uses_correct_cache_keys(self, mock_job_class, mock_get_key "vc_detection_enabled": True, "cached_at": "2026-01-20T10:00:00Z", "cache_timeout": 600, + "use_sysname": True, + "strip_domain": False, } mock_job_class.objects.get.return_value = mock_job @@ -708,12 +710,16 @@ def test_load_success_uses_correct_cache_keys(self, mock_job_class, mock_get_key filters={"location": "dc1"}, device_id=1, vc_enabled=True, + use_sysname=True, + strip_domain=False, ) mock_get_key.assert_any_call( server_key="primary", filters={"location": "dc1"}, device_id=2, vc_enabled=True, + use_sysname=True, + strip_domain=False, ) assert len(results) == 2 @@ -734,6 +740,8 @@ def test_load_extracts_filters_from_job_data(self, mock_job_class, mock_get_key, "vc_detection_enabled": False, "cached_at": "2026-01-20T10:00:00Z", "cache_timeout": 300, + "use_sysname": True, + "strip_domain": False, } mock_job_class.objects.get.return_value = mock_job mock_get_key.return_value = "test_key" @@ -748,6 +756,8 @@ def test_load_extracts_filters_from_job_data(self, mock_job_class, mock_get_key, filters={"location": "dc2", "type": "router"}, device_id=1, vc_enabled=False, + use_sysname=True, + strip_domain=False, ) @patch("netbox_librenms_plugin.views.imports.list.cache") diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 96ed7ba62f..3ef6204362 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -64,6 +64,31 @@ def test_get_import_device_cache_key(self): assert "secondary" in key assert "456" in key + def test_validated_device_cache_key_unique_per_naming_mode(self): + """Different naming preferences produce different cache keys.""" + from netbox_librenms_plugin.import_utils import get_validated_device_cache_key + + base_args = dict(server_key="default", filters={}, device_id=123, vc_enabled=False) + key_default = get_validated_device_cache_key(**base_args) + key_no_sysname = get_validated_device_cache_key(**base_args, use_sysname=False) + key_strip = get_validated_device_cache_key(**base_args, strip_domain=True) + + assert key_default != key_no_sysname + assert key_default != key_strip + assert key_no_sysname != key_strip + + def test_cache_metadata_key_unique_per_naming_mode(self): + """Different naming preferences produce different metadata cache keys.""" + from netbox_librenms_plugin.import_utils import get_cache_metadata_key + + base_args = dict(server_key="default", filters={}, vc_enabled=False) + key_default = get_cache_metadata_key(**base_args) + key_no_sysname = get_cache_metadata_key(**base_args, use_sysname=False) + key_strip = get_cache_metadata_key(**base_args, strip_domain=True) + + assert key_default != key_no_sysname + assert key_default != key_strip + # ============================================================================= # TestDeviceNameDetermination - 6 tests @@ -149,8 +174,8 @@ def test_determine_device_name_fallback_to_device_id(self): class TestDeviceRetrieval: """Test device retrieval and filtering functions.""" - @patch("netbox_librenms_plugin.import_utils.cache") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.filters.cache") + @patch("netbox_librenms_plugin.import_utils.filters.LibreNMSAPI") def test_get_librenms_devices_for_import_success(self, mock_api_class, mock_cache): """Retrieve devices from LibreNMS API.""" mock_cache.get.return_value = None # Cache miss @@ -171,7 +196,7 @@ def test_get_librenms_devices_for_import_success(self, mock_api_class, mock_cach assert len(devices) == 2 assert devices[0]["hostname"] == "switch-01" - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_librenms_devices_for_import_uses_cache(self, mock_cache): """Cached results returned on repeat call.""" cached_devices = [ @@ -189,7 +214,7 @@ def test_get_librenms_devices_for_import_uses_cache(self, mock_cache): assert devices[0]["hostname"] == "cached-device" mock_api.list_devices.assert_not_called() - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_librenms_devices_for_import_cache_miss(self, mock_cache): """API called when cache empty.""" mock_cache.get.return_value = None @@ -209,7 +234,7 @@ def test_get_librenms_devices_for_import_cache_miss(self, mock_cache): mock_api.list_devices.assert_called_once() assert len(devices) == 1 - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_device_count_for_filters_success(self, mock_cache): """Returns correct count from API.""" mock_cache.get.return_value = [ @@ -225,13 +250,13 @@ def test_get_device_count_for_filters_success(self, mock_cache): assert count == 3 - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_device_count_excludes_disabled(self, mock_cache): - """Count respects show_disabled filter parameter.""" + """Count respects show_disabled filter parameter: disabled==1 devices excluded.""" mock_cache.get.return_value = [ - {"device_id": 1, "hostname": "switch-01", "status": 1}, - {"device_id": 2, "hostname": "switch-02", "status": 1}, - {"device_id": 3, "hostname": "switch-03", "status": 0}, # disabled + {"device_id": 1, "hostname": "switch-01", "disabled": 0, "status": 1}, + {"device_id": 2, "hostname": "switch-02", "disabled": 0, "status": 0}, + {"device_id": 3, "hostname": "switch-03", "disabled": 1, "status": 1}, # disabled in LibreNMS ] mock_api = MagicMock() @@ -279,7 +304,7 @@ def test_empty_virtual_chassis_data(self): assert data["members"] == [] assert data["detection_error"] is None - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.virtual_chassis.cache") def test_get_virtual_chassis_data_returns_empty_without_api(self, mock_cache): """Get VC data returns empty structure without API.""" from netbox_librenms_plugin.import_utils import get_virtual_chassis_data @@ -299,14 +324,14 @@ class TestDeviceValidation: """Test device validation for import.""" @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_site_match_found( self, mock_site_model, @@ -341,7 +366,6 @@ def test_validate_device_site_match_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [mock_site] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -358,14 +382,14 @@ def test_validate_device_site_match_found( assert result["site"]["site"] == mock_site @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_site_not_found( self, mock_site_model, @@ -399,7 +423,6 @@ def test_validate_device_site_not_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -416,15 +439,15 @@ def test_validate_device_site_not_found( assert any("site" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") - @patch("netbox_librenms_plugin.import_utils.DeviceType") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceType") def test_validate_device_platform_match_found( self, mock_device_type, @@ -461,7 +484,6 @@ def test_validate_device_platform_match_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -478,14 +500,14 @@ def test_validate_device_platform_match_found( assert result["platform"]["platform"] == mock_platform @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_platform_not_found( self, mock_site_model, @@ -519,7 +541,6 @@ def test_validate_device_platform_not_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -535,14 +556,14 @@ def test_validate_device_platform_not_found( assert result["platform"]["found"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_type_match_found( self, mock_site_model, @@ -577,7 +598,6 @@ def test_validate_device_type_match_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -594,15 +614,15 @@ def test_validate_device_type_match_found( assert result["device_type"]["device_type"] == mock_dt @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") - @patch("netbox_librenms_plugin.import_utils.DeviceType") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceType") def test_validate_device_type_not_found( self, mock_device_type, @@ -638,7 +658,6 @@ def test_validate_device_type_not_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -651,18 +670,18 @@ def test_validate_device_type_not_found( result = validate_device_for_import(device_data, include_vc_detection=False) - assert result["device_type"]["matched"] is False + assert result["device_type"]["found"] is False assert any("device type" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_role_required( self, mock_site_model, @@ -698,7 +717,6 @@ def test_validate_device_role_required( } mock_role.objects.all.return_value = [MagicMock(id=1, name="Access Switch")] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [mock_site] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -716,14 +734,14 @@ def test_validate_device_role_required( assert any("role" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_handles_empty_location( self, mock_site_model, @@ -757,7 +775,6 @@ def test_validate_device_handles_empty_location( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -775,14 +792,14 @@ def test_validate_device_handles_empty_location( assert result["site"]["found"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_handles_empty_os( self, mock_site_model, @@ -816,7 +833,6 @@ def test_validate_device_handles_empty_os( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -833,15 +849,15 @@ def test_validate_device_handles_empty_os( assert result["platform"]["found"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") - @patch("netbox_librenms_plugin.import_utils.DeviceType") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceType") def test_validate_device_handles_empty_hardware( self, mock_device_type, @@ -877,7 +893,6 @@ def test_validate_device_handles_empty_hardware( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -891,17 +906,17 @@ def test_validate_device_handles_empty_hardware( result = validate_device_for_import(device_data, include_vc_detection=False) assert result is not None - assert result["device_type"]["matched"] is False + assert result["device_type"]["found"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_duplicate_detection( self, mock_site_model, @@ -933,14 +948,14 @@ def test_validate_device_duplicate_detection( assert result["can_import"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_returns_complete_state( self, mock_site_model, @@ -974,7 +989,6 @@ def test_validate_device_returns_complete_state( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -999,29 +1013,27 @@ def test_validate_device_returns_complete_state( assert "cluster" in result assert "platform" in result - @patch("netbox_librenms_plugin.import_utils.cache") - @patch("virtualization.models.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.cache") @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_import_as_vm( self, mock_site_model, mock_rack, - mock_cluster_module, + mock_cluster, mock_role, mock_match_type, mock_find_platform, mock_find_site, mock_device, mock_vm, - mock_cluster_local, mock_cache, ): """Import as VM mode uses cluster instead of site/device_type.""" @@ -1045,10 +1057,8 @@ def test_validate_device_import_as_vm( } mock_role.objects.all.return_value = [] mock_clusters = [MagicMock(id=1, name="VMware Cluster")] - # Cluster is imported locally in the VM path, so we need to mock it there - mock_cluster_local.objects.all.return_value = mock_clusters + mock_cluster.objects.all.return_value = mock_clusters mock_cache.get.return_value = None # Force cache miss to trigger Cluster.objects.all() - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1064,14 +1074,14 @@ def test_validate_device_import_as_vm( assert result["cluster"]["available_clusters"] == mock_clusters @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_existing_vm_blocks_import( self, mock_site_model, @@ -1102,3 +1112,2436 @@ def test_validate_device_existing_vm_blocks_import( assert result["existing_device"] == existing_vm assert result["can_import"] is False assert result["import_as_vm"] is True + + +class TestSerialNumberMatching: + """Test serial number matching in device validation.""" + + SERIAL_PATCHES = [ + "netbox_librenms_plugin.import_utils.device_operations.Site", + "netbox_librenms_plugin.import_utils.device_operations.Rack", + "netbox_librenms_plugin.import_utils.device_operations.Cluster", + "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", + "netbox_librenms_plugin.import_utils.device_operations.DeviceType", + "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", + "netbox_librenms_plugin.import_utils.device_operations.Device", + "virtualization.models.VirtualMachine", + ] + + def _start_patches(self): + """Start all common patches and return mocks in standard order.""" + self._patchers = [patch(p) for p in self.SERIAL_PATCHES] + mocks = [p.start() for p in self._patchers] + ( + self.mock_site_model, + self.mock_rack, + self.mock_cluster, + self.mock_role, + self.mock_device_type, + self.mock_match_type, + self.mock_find_platform, + self.mock_find_site, + self.mock_device, + self.mock_vm, + ) = mocks + self.mock_device_type.objects.all.return_value = [] + + def _stop_patches(self): + """Stop all patches.""" + for p in self._patchers: + p.stop() + + def setup_method(self): + """Set up common patches for serial number tests.""" + self._start_patches() + + def teardown_method(self): + """Tear down patches.""" + self._stop_patches() + + def test_serial_match_blocks_import(self): + """Device with matching serial blocks import.""" + existing = MagicMock() + existing.name = "existing-device" + existing.serial = "ABC123" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "serial" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "new-hostname", "serial": "ABC123"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["can_import"] is False + assert result["existing_match_type"] == "serial" + assert result["existing_device"] == existing + + def test_serial_match_same_hostname_offers_link(self): + """Serial + hostname match offers link action.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "ABC123" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "serial" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "serial": "ABC123"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["serial_action"] == "link" + assert result["existing_match_type"] == "serial" + assert "not linked to LibreNMS" in result["warnings"][0] + + def test_serial_match_diff_hostname_offers_hostname_differs(self): + """Serial matches but hostname differs offers hostname_differs action.""" + existing = MagicMock() + existing.name = "old-hostname" + existing.serial = "ABC123" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "serial" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "new-hostname", "serial": "ABC123"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["serial_action"] == "hostname_differs" + assert result["existing_match_type"] == "serial" + assert "hostname differs" in result["warnings"][0] + + def test_hostname_match_diff_serial_offers_update(self): + """Hostname matches but serial differs offers update_serial action.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "OLD_SERIAL" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "name__iexact" in kwargs: + result.first.return_value = existing + elif "serial" in kwargs: + result.first.return_value = None + result.exclude.return_value.first.return_value = None + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "serial": "NEW_SERIAL"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["serial_action"] == "update_serial" + assert result["existing_match_type"] == "hostname" + assert "Hardware may have been replaced" in result["warnings"][0] + + def _setup_no_match_mocks(self): + """Configure mocks for tests where no device match is expected.""" + self.mock_vm.objects.filter.return_value.first.return_value = None + self.mock_device.objects.filter.return_value.first.return_value = None + self.mock_find_site.return_value = {"found": False, "site": None, "match_type": None, "confidence": 0} + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_site_model.objects.all.return_value = [] + + def test_serial_dash_ignored(self): + """Serial '-' is not treated as a match.""" + self._setup_no_match_mocks() + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "serial": "-"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_match_type"] is None + assert result["serial_action"] is None + + def test_serial_empty_ignored(self): + """Empty serial skips serial matching.""" + self._setup_no_match_mocks() + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "serial": ""} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_match_type"] is None + assert result["serial_action"] is None + + def test_serial_none_ignored(self): + """None serial skips serial matching.""" + self._setup_no_match_mocks() + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "serial": None} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_match_type"] is None + assert result["serial_action"] is None + + def test_hostname_match_serial_conflict_warns(self): + """Hostname matches, incoming serial already on another device warns about conflict.""" + hostname_device = MagicMock() + hostname_device.name = "switch-01" + hostname_device.serial = "OLD_SERIAL" + + serial_conflict_device = MagicMock() + serial_conflict_device.name = "other-device" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "name__iexact" in kwargs: + result.first.return_value = hostname_device + elif "serial" in kwargs: + result.exclude.return_value.first.return_value = serial_conflict_device + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "serial": "CONFLICTING_SERIAL"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["serial_action"] == "conflict" + assert result["existing_match_type"] == "hostname" + assert "Serial conflict" in result["warnings"][0] + + def test_librenms_id_match_shows_serial_confirmed(self): + """librenms_id match with matching serial shows confirmation.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "ABC123" + existing.virtual_chassis = None # Not a VC member β†’ use plain hostname comparison + existing.vc_position = None + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(*args, **kwargs): + result = MagicMock() + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + if q_has_librenms: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + self.mock_find_site.return_value = { + "found": True, + "site": MagicMock(), + "match_type": "exact", + "confidence": 1.0, + } + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_site_model.objects.all.return_value = [] + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "sysName": "switch-01", "serial": "ABC123"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_match_type"] == "librenms_id" + assert result["can_import"] is False + assert result["serial_confirmed"] is True + assert result["name_matches"] is True + + def test_librenms_id_match_detects_serial_drift(self): + """librenms_id match with different serial warns about drift.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "OLD_SERIAL" + existing.virtual_chassis = None + existing.vc_position = None + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(*args, **kwargs): + result = MagicMock() + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + if q_has_librenms: + result.first.return_value = existing + elif "serial" in kwargs: + result.first.return_value = None + result.exclude.return_value.first.return_value = None + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + self.mock_find_site.return_value = { + "found": True, + "site": MagicMock(), + "match_type": "exact", + "confidence": 1.0, + } + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_site_model.objects.all.return_value = [] + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "serial": "NEW_SERIAL"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_match_type"] == "librenms_id" + assert result["serial_action"] == "update_serial" + assert any("Hardware may have been replaced" in w for w in result["warnings"]) + + def test_librenms_id_match_still_validates_site(self): + """librenms_id match continues to populate site/type validation.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "" + existing.virtual_chassis = None + existing.vc_position = None + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(*args, **kwargs): + result = MagicMock() + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + if q_has_librenms: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + mock_site = MagicMock(id=1, name="DC1") + self.mock_find_site.return_value = {"found": True, "site": mock_site, "match_type": "exact", "confidence": 1.0} + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + mock_dt = MagicMock() + self.mock_match_type.return_value = {"matched": True, "device_type": mock_dt, "match_type": "exact"} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_site_model.objects.all.return_value = [] + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = {"device_id": 1, "hostname": "switch-01", "location": "DC1", "hardware": "WS-C4900M"} + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_match_type"] == "librenms_id" + assert result["can_import"] is False + assert result["is_ready"] is False + # Site and device_type should still be populated + assert result["site"]["found"] is True + assert result["site"]["site"] == mock_site + assert result["device_type"]["found"] is True + + def test_existing_device_role_populated(self): + """Existing device's role should be shown in validation details.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "ABC123" + existing.virtual_chassis = None + existing.vc_position = None + mock_existing_role = MagicMock() + mock_existing_role.name = "Access Switch" + existing.role = mock_existing_role + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "serial" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + self.mock_find_site.return_value = {"found": False, "site": None, "match_type": None, "confidence": 0} + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": True, "device_type": MagicMock(), "match_type": "exact"} + self.mock_role.objects.all.return_value = [mock_existing_role] + self.mock_cluster.objects.all.return_value = [] + self.mock_site_model.objects.all.return_value = [] + + with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: + mock_cache.get.return_value = None + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "switch-01", + "serial": "ABC123", + "location": "", + "hardware": "", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_device"] == existing + assert result["device_role"]["found"] is True + assert result["device_role"]["role"] == mock_existing_role + + def test_device_type_mismatch_flagged(self): + """Device type mismatch between existing device and LibreNMS should be flagged.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "ABC123" + existing.virtual_chassis = None + existing.vc_position = None + existing_device_type = MagicMock() + existing_device_type.pk = 1 + existing_device_type.__str__ = lambda self: "Old Type" + existing.device_type = existing_device_type + existing.role = MagicMock() + + librenms_device_type = MagicMock() + librenms_device_type.pk = 2 + librenms_device_type.__str__ = lambda self: "New Type" + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "serial" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + self.mock_find_site.return_value = {"found": False, "site": None, "match_type": None, "confidence": 0} + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = { + "matched": True, + "device_type": librenms_device_type, + "match_type": "exact", + } + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_site_model.objects.all.return_value = [] + + with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: + mock_cache.get.return_value = None + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "switch-01", + "serial": "ABC123", + "location": "", + "hardware": "New Type", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["device_type_mismatch"] is True + assert any("Device type mismatch" in w for w in result["warnings"]) + + def test_no_device_type_mismatch_when_types_match(self): + """No mismatch flag when existing device type matches LibreNMS.""" + existing = MagicMock() + existing.name = "switch-01" + existing.serial = "ABC123" + existing.virtual_chassis = None + existing.vc_position = None + same_device_type = MagicMock() + same_device_type.pk = 1 + existing.device_type = same_device_type + existing.role = MagicMock() + + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "serial" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + self.mock_find_site.return_value = {"found": False, "site": None, "match_type": None, "confidence": 0} + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": True, "device_type": same_device_type, "match_type": "exact"} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_site_model.objects.all.return_value = [] + + with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: + mock_cache.get.return_value = None + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "switch-01", + "serial": "ABC123", + "location": "", + "hardware": "Same Type", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["device_type_mismatch"] is False + + +class TestNameMatchesWithNamingPreferences: + """Test VC-aware name matching with use_sysname/strip_domain preferences.""" + + PATCHES = [ + "netbox_librenms_plugin.import_utils.device_operations.Site", + "netbox_librenms_plugin.import_utils.device_operations.Rack", + "netbox_librenms_plugin.import_utils.device_operations.Cluster", + "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", + "netbox_librenms_plugin.import_utils.device_operations.DeviceType", + "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", + "netbox_librenms_plugin.import_utils.device_operations.Device", + "virtualization.models.VirtualMachine", + ] + + def setup_method(self): + self._patchers = [patch(p) for p in self.PATCHES] + mocks = [p.start() for p in self._patchers] + ( + self.mock_site, + self.mock_rack, + self.mock_cluster, + self.mock_role, + self.mock_device_type, + self.mock_match_type, + self.mock_find_platform, + self.mock_find_site, + self.mock_device, + self.mock_vm, + ) = mocks + self.mock_device_type.objects.all.return_value = [] + self.mock_vm.objects.filter.return_value.first.return_value = None + self.mock_find_site.return_value = { + "found": True, + "site": MagicMock(), + "match_type": "exact", + "confidence": 1.0, + } + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_site.objects.all.return_value = [] + + def teardown_method(self): + for p in self._patchers: + p.stop() + + def _make_existing(self, name, serial="SN001", virtual_chassis=None, vc_position=None): + existing = MagicMock() + existing.name = name + existing.serial = serial + existing.virtual_chassis = virtual_chassis + existing.vc_position = vc_position + existing.custom_field_data = {"librenms_id": {"default": 42}} + return existing + + def _setup_librenms_id_filter(self, existing): + def device_filter(*args, **kwargs): + result = MagicMock() + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + result.first.return_value = existing if q_has_librenms else None + result.exclude.return_value.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + def test_strip_domain_name_matches(self): + """strip_domain=True resolves 'switch-01.example.com' to 'switch-01', matching existing device.""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + + existing = self._make_existing("switch-01") + self._setup_librenms_id_filter(existing) + + device_data = { + "device_id": 42, + "hostname": "switch-01.example.com", + "sysName": "switch-01.example.com", + "serial": "SN001", + } + result = validate_device_for_import(device_data, include_vc_detection=False, strip_domain=True) + assert result["name_matches"] is True + assert result["name_sync_available"] is False + + def test_sysname_disabled_uses_hostname(self): + """use_sysname=False falls back to hostname for name comparison.""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + + existing = self._make_existing("switch-hostname") + self._setup_librenms_id_filter(existing) + + device_data = { + "device_id": 42, + "hostname": "switch-hostname", + "sysName": "switch-sysname", + "serial": "SN001", + } + result = validate_device_for_import(device_data, include_vc_detection=False, use_sysname=False) + assert result["name_matches"] is True + assert result["resolved_name"] == "switch-hostname" + + def test_name_mismatch_offers_sync(self): + """When resolved name differs from existing device name, name_sync_available is set.""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + + existing = self._make_existing("old-name") + self._setup_librenms_id_filter(existing) + + device_data = { + "device_id": 42, + "hostname": "new-name", + "sysName": "new-name", + "serial": "SN001", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + assert result["name_matches"] is False + assert result["name_sync_available"] is True + assert result["suggested_name"] == "new-name" + + def test_vc_member_name_matches(self): + """Existing VC member name is compared against vc_member_name(hostname, vc_position).""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + from netbox_librenms_plugin.import_utils.virtual_chassis import _generate_vc_member_name + + mock_vc = MagicMock() + expected_name = _generate_vc_member_name("stack-master", 2, serial="SN001") + existing = self._make_existing(expected_name, serial="SN001", virtual_chassis=mock_vc, vc_position=2) + self._setup_librenms_id_filter(existing) + + device_data = { + "device_id": 42, + "hostname": "stack-master", + "sysName": "stack-master", + "serial": "SN001", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + assert result["name_matches"] is True + + def test_vc_member_name_mismatch_suggests_vc_name(self): + """When VC member name differs, suggested_name is the expected VC member name.""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + from netbox_librenms_plugin.import_utils.virtual_chassis import _generate_vc_member_name + + mock_vc = MagicMock() + existing = self._make_existing("wrong-name", serial="SN001", virtual_chassis=mock_vc, vc_position=2) + self._setup_librenms_id_filter(existing) + + device_data = { + "device_id": 42, + "hostname": "stack-master", + "sysName": "stack-master", + "serial": "SN001", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + expected_name = _generate_vc_member_name("stack-master", 2, serial="SN001") + assert result["name_matches"] is False + assert result["name_sync_available"] is True + assert result["suggested_name"] == expected_name + + def test_vc_member_with_strip_domain(self): + """strip_domain applies before VC member name comparison.""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + from netbox_librenms_plugin.import_utils.virtual_chassis import _generate_vc_member_name + + mock_vc = MagicMock() + expected_name = _generate_vc_member_name("stack", 1, serial="SN001") + existing = self._make_existing(expected_name, serial="SN001", virtual_chassis=mock_vc, vc_position=1) + self._setup_librenms_id_filter(existing) + + device_data = { + "device_id": 42, + "hostname": "stack.example.com", + "sysName": "stack.example.com", + "serial": "SN001", + } + result = validate_device_for_import(device_data, include_vc_detection=False, strip_domain=True) + assert result["name_matches"] is True + + def test_naming_criteria_populated(self): + """naming_criteria dict is set in result with use_sysname/strip_domain/source.""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + + self.mock_device.objects.filter.return_value.first.return_value = None + + device_data = { + "device_id": 99, + "hostname": "router-01", + "sysName": "router-sysname", + } + result = validate_device_for_import( + device_data, include_vc_detection=False, use_sysname=True, strip_domain=False + ) + criteria = result["naming_criteria"] + assert criteria is not None + assert criteria["use_sysname"] is True + assert criteria["strip_domain"] is False + assert criteria["raw_sysname"] == "router-sysname" + assert criteria["raw_hostname"] == "router-01" + assert criteria["source"] == "sysname" + + def test_naming_criteria_source_hostname_when_sysname_disabled(self): + """naming_criteria source is 'hostname' when use_sysname=False.""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + + self.mock_device.objects.filter.return_value.first.return_value = None + + device_data = { + "device_id": 99, + "hostname": "router-01", + "sysName": "router-sysname", + } + result = validate_device_for_import( + device_data, include_vc_detection=False, use_sysname=False, strip_domain=False + ) + assert result["naming_criteria"]["source"] == "hostname" + + +class TestLegacyLibreNMSIdMigration: + """Test detection of legacy bare-integer librenms_id format during device validation.""" + + PATCHES = [ + "netbox_librenms_plugin.import_utils.device_operations.Site", + "netbox_librenms_plugin.import_utils.device_operations.Rack", + "netbox_librenms_plugin.import_utils.device_operations.Cluster", + "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", + "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", + "netbox_librenms_plugin.import_utils.device_operations.Device", + "virtualization.models.VirtualMachine", + ] + + def setup_method(self): + self._patchers = [patch(p) for p in self.PATCHES] + mocks = [p.start() for p in self._patchers] + ( + self.mock_site_model, + self.mock_rack, + self.mock_cluster, + self.mock_role, + self.mock_match_type, + self.mock_find_platform, + self.mock_find_site, + self.mock_device, + self.mock_vm, + ) = mocks + + self.mock_find_site.return_value = { + "found": True, + "site": MagicMock(), + "match_type": "exact", + "confidence": 1.0, + } + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_site_model.objects.all.return_value = [] + self.mock_vm.objects.filter.return_value.first.return_value = None + + def teardown_method(self): + for p in self._patchers: + p.stop() + + def _make_existing(self, librenms_id_value, serial="SN001"): + existing = MagicMock() + existing.name = "switch-01" + existing.serial = serial + existing.virtual_chassis = None + existing.vc_position = None + existing.custom_field_data = {"librenms_id": librenms_id_value} + return existing + + def _setup_device_filter(self, existing): + def device_filter(*args, **kwargs): + result = MagicMock() + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + result.first.return_value = existing if q_has_librenms else None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + def test_legacy_int_sets_needs_migration_flag(self): + """Device with bare-integer librenms_id sets librenms_id_needs_migration=True.""" + existing = self._make_existing(librenms_id_value=42, serial="SN001") + self._setup_device_filter(existing) + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + result = validate_device_for_import( + {"device_id": 42, "hostname": "switch-01", "serial": "SN001"}, + include_vc_detection=False, + ) + + assert result["existing_match_type"] == "librenms_id" + assert result["librenms_id_needs_migration"] is True + assert result["serial_confirmed"] is True + + def test_legacy_int_no_serial_still_sets_flag(self): + """Legacy int format sets the migration flag even when serial is absent.""" + existing = self._make_existing(librenms_id_value=42, serial="") + self._setup_device_filter(existing) + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + result = validate_device_for_import( + {"device_id": 42, "hostname": "switch-01"}, + include_vc_detection=False, + ) + + assert result["librenms_id_needs_migration"] is True + assert result["serial_confirmed"] is False + + def test_json_format_does_not_set_flag(self): + """Device with JSON librenms_id does NOT set librenms_id_needs_migration.""" + existing = self._make_existing(librenms_id_value={"default": 42}, serial="SN001") + self._setup_device_filter(existing) + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + result = validate_device_for_import( + {"device_id": 42, "hostname": "switch-01", "serial": "SN001"}, + include_vc_detection=False, + ) + + assert result["existing_match_type"] == "librenms_id" + assert result["librenms_id_needs_migration"] is False + + def test_migrate_legacy_librenms_id_helper(self): + """migrate_legacy_librenms_id converts int to {server_key: int}.""" + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + obj = MagicMock() + obj.custom_field_data = {"librenms_id": 42} + result = migrate_legacy_librenms_id(obj, "primary") + + assert result is True + assert obj.custom_field_data["librenms_id"] == {"primary": 42} + + def test_migrate_legacy_librenms_id_noop_for_json(self): + """migrate_legacy_librenms_id is a no-op when value is already a dict.""" + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + obj = MagicMock() + obj.custom_field_data = {"librenms_id": {"primary": 42}} + result = migrate_legacy_librenms_id(obj, "primary") + + assert result is False + assert obj.custom_field_data["librenms_id"] == {"primary": 42} + + def test_migrate_legacy_librenms_id_noop_for_none(self): + """migrate_legacy_librenms_id is a no-op when librenms_id is absent.""" + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + obj = MagicMock() + obj.custom_field_data = {} + result = migrate_legacy_librenms_id(obj, "primary") + + assert result is False + + +class TestDeviceConflictActionView: + """Test DeviceConflictActionView conflict resolution actions.""" + + def _create_view(self): + """Create a DeviceConflictActionView instance with mocked dependencies.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = DeviceConflictActionView() + view._librenms_api = MagicMock() + view._librenms_api.server_key = "default" + view.request = MagicMock() + view.request.user.has_perm.return_value = True + return view + + def _create_request(self, action, existing_device_id, use_sysname=False, strip_domain=False): + """Create a mock request with POST data.""" + request = MagicMock() + # Always include both toggles so _resolve_naming_preferences never falls through + # to the user-pref/settings DB path, which would hit the real database. + post_data = { + "action": action, + "existing_device_id": str(existing_device_id), + "use-sysname-toggle": "on" if use_sysname else "off", + "strip-domain-toggle": "on" if strip_domain else "off", + } + request.POST = post_data + return request + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache): + """Link action should set librenms_id and update name from sysName.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {} + existing_device.name = "84.116.251.35" + + libre_device = { + "device_id": 10, + "hostname": "84.116.251.35", + "sysName": "switch-01.example.com", + "serial": "ABC123", + } + validation = {"can_import": False, "existing_device": existing_device} + selections = {} + + request = self._create_request("link", 42, use_sysname=True) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, + ): + mock_tx.atomic.return_value = MagicMock() + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} + assert existing_device.name == "switch-01.example.com" + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_update_action_sets_hostname_serial_and_librenms_id(self, mock_cache_key, mock_cache): + """Update action should set hostname, serial, and librenms_id.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {} + existing_device.name = "old-name" + existing_device.serial = "OLD-SERIAL" + + libre_device = { + "device_id": 10, + "hostname": "84.116.251.35", + "sysName": "new-name.example.com", + "serial": "NEW-SERIAL", + } + validation = {"can_import": False, "existing_device": existing_device} + selections = {} + + request = self._create_request("update", 42, use_sysname=True) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, + ): + mock_tx.atomic.return_value = MagicMock() + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} + assert existing_device.serial == "NEW-SERIAL" + assert existing_device.name == "new-name.example.com" + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_update_serial_action_updates_serial_only(self, mock_cache_key, mock_cache): + """Update serial action should update serial and librenms_id but not hostname.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {} + existing_device.name = "switch-01" + existing_device.serial = "OLD-SERIAL" + + libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "NEW-SERIAL"} + validation = {"can_import": False, "existing_device": existing_device} + selections = {} + + request = self._create_request("update_serial", 42) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, + ): + mock_tx.atomic.return_value = MagicMock() + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} + assert existing_device.serial == "NEW-SERIAL" + # Name should NOT be changed by update_serial + assert existing_device.name == "switch-01" + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_update_skips_dash_serial(self, mock_cache_key, mock_cache): + """Update should not set serial to '-' (LibreNMS placeholder).""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {} + existing_device.name = "switch-01" + existing_device.serial = "EXISTING" + + libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "-"} + validation = {"can_import": False, "existing_device": existing_device} + selections = {} + + request = self._create_request("update_serial", 42) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, + ): + mock_tx.atomic.return_value = MagicMock() + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + # Serial should NOT be updated to '-' + assert existing_device.serial == "EXISTING" + + def test_missing_action_returns_400(self): + """Missing action or existing_device_id should return 400.""" + view = self._create_view() + request = MagicMock() + request.POST = {} + + response = view.post(request, device_id=10) + assert response.status_code == 400 + + def test_unknown_action_returns_400(self): + """Unknown action should return 400.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + request = self._create_request("invalid_action", 42) + + existing_device = MagicMock() + existing_device.pk = 42 + libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "ABC"} + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "require_object_permissions", return_value=None), + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + # Include existing_device so the validated-conflict-target guard passes; + # we want to exercise the unknown-action branch, not the missing-device guard. + mock_validate.return_value = (libre_device, {"existing_device": existing_device}, {}) + + response = view.post(request, device_id=10) + + assert response.status_code == 400 + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_sync_name_action_updates_name(self, mock_cache_key, mock_cache): + """Sync name action should update device name using sysName.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {"librenms_id": 10} + existing_device.name = "84.116.251.35" + + libre_device = { + "device_id": 10, + "hostname": "84.116.251.35", + "sysName": "switch-01.example.com", + "serial": "ABC123", + } + validation = {"can_import": False, "existing_device": existing_device} + selections = {} + + request = self._create_request("sync_name", 42, use_sysname=True) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.name == "switch-01.example.com" + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_device_type_mismatch_blocked_without_force(self, mock_cache_key, mock_cache): + """Action should be blocked when device_type_mismatch is True and force is not set.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {} + + libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "ABC123"} + validation = {"can_import": False, "device_type_mismatch": True, "existing_device": existing_device} + selections = {} + + request = self._create_request("link", 42, use_sysname=True) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_validate.return_value = (libre_device, validation, selections) + + response = view.post(request, device_id=10) + + assert response.status_code == 400 + existing_device.save.assert_not_called() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cache): + """Action should proceed when device_type_mismatch is True and force is set.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {} + existing_device.name = "old-name" + + libre_device = { + "device_id": 10, + "hostname": "switch-01", + "sysName": "switch-01.example.com", + "serial": "ABC123", + } + validation = {"can_import": False, "device_type_mismatch": True, "existing_device": existing_device} + selections = {} + + request = self._create_request("link", 42, use_sysname=True) + request.POST["force"] = "on" + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, + ): + mock_tx.atomic.return_value = MagicMock() + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cache): + """Force with device_type_mismatch should update existing device's device_type.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {} + existing_device.name = "old-name" + + librenms_device_type = MagicMock() + librenms_device_type.pk = 99 + libre_device = { + "device_id": 10, + "hostname": "switch-01", + "sysName": "switch-01.example.com", + "serial": "ABC123", + } + validation = { + "can_import": False, + "device_type_mismatch": True, + "device_type": {"device_type": librenms_device_type}, + "existing_device": existing_device, + } + selections = {} + + request = self._create_request("link", 42, use_sysname=True) + request.POST["force"] = "on" + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, + ): + mock_tx.atomic.return_value = MagicMock() + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.device_type == librenms_device_type + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_update_type_action_changes_device_type(self, mock_cache_key, mock_cache): + """update_type action should change device type on existing device.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.pk = 42 + existing_device.custom_field_data = {"librenms_id": 10} + existing_device.name = "switch-01" + old_device_type = MagicMock() + existing_device.device_type = old_device_type + + new_device_type = MagicMock() + new_device_type.pk = 99 + libre_device = { + "device_id": 10, + "hostname": "switch-01", + "serial": "ABC123", + } + validation = { + "can_import": False, + "device_type_mismatch": True, + "device_type": {"device_type": new_device_type}, + "existing_device": existing_device, + } + selections = {} + + request = self._create_request("update_type", 42) + request.POST["force"] = "on" + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.device_type == new_device_type + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_sync_serial_action(self, mock_cache_key, mock_cache): + """sync_serial action should update serial from LibreNMS.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.serial = "OLD123" + libre_device = {"device_id": 10, "serial": "NEW456", "sysName": "test"} + validation = {"existing_device": existing_device, "device_type_mismatch": False} + selections = {} + + request = self._create_request("sync_serial", 42) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, + ): + mock_tx.atomic.return_value = MagicMock() + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.serial == "NEW456" + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_sync_platform_action(self, mock_cache_key, mock_cache): + """sync_platform action should update platform from LibreNMS OS.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + existing_device.platform = None + libre_device = {"device_id": 10, "os": "ios", "sysName": "test"} + validation = {"existing_device": existing_device, "device_type_mismatch": False} + selections = {} + + mock_platform = MagicMock() + request = self._create_request("sync_platform", 42) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + # Patch find_matching_platform at the utility module level β€” the action imports + # it from netbox_librenms_plugin.utils, so that is the correct seam to mock. + patch("netbox_librenms_plugin.utils.find_matching_platform") as mock_find_platform, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_find_platform.return_value = {"found": True, "platform": mock_platform, "match_type": "exact"} + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.platform == mock_platform + existing_device.save.assert_called_once() + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_sync_device_type_action(self, mock_cache_key, mock_cache): + """sync_device_type action should update device type from LibreNMS hardware match.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() + existing_device = MagicMock() + new_device_type = MagicMock() + libre_device = {"device_id": 10, "hardware": "Catalyst C4900M", "sysName": "test"} + validation = {"existing_device": existing_device, "device_type_mismatch": False} + selections = {} + + request = self._create_request("sync_device_type", 42) + + with ( + patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "render_device_row") as mock_render, + patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.utils.match_librenms_hardware_to_device_type") as mock_hw_match, + ): + mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_hw_match.return_value = {"matched": True, "device_type": new_device_type} + mock_validate.return_value = (libre_device, validation, selections) + mock_render.return_value = MagicMock() + + view.post(request, device_id=10) + + assert existing_device.device_type == new_device_type + existing_device.save.assert_called_once() + + +class TestBuildSyncInfo: + """Test DeviceValidationDetailsView._build_sync_info method.""" + + def test_all_synced(self): + """When serial, platform, device type all match, all_synced is True.""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "ABC123" + platform = MagicMock() + platform.pk = 1 + existing.platform = platform + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + libre_device = {"serial": "ABC123", "os": "ios", "hardware": "Catalyst C4900M"} + + with ( + patch("netbox_librenms_plugin.utils.find_matching_platform") as mock_platform_match, + patch("netbox_librenms_plugin.utils.match_librenms_hardware_to_device_type") as mock_hw_match, + ): + mock_platform_match.return_value = {"found": True, "platform": platform} + mock_hw_match.return_value = {"matched": True, "device_type": device_type} + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + assert result["all_synced"] is True + assert result["serial_synced"] is True + assert result["platform_synced"] is True + assert result["device_type_synced"] is True + + def test_serial_out_of_sync(self): + """When serial differs, serial_synced is False.""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "OLD123" + existing.platform = None + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + libre_device = {"serial": "NEW456", "os": "-", "hardware": "-"} + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + assert result["serial_synced"] is False + assert result["all_synced"] is False + + def test_platform_out_of_sync(self): + """When platform differs, platform_synced is False.""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "ABC123" + old_platform = MagicMock() + old_platform.pk = 1 + existing.platform = old_platform + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + new_platform = MagicMock() + new_platform.pk = 2 + + libre_device = {"serial": "ABC123", "os": "junos", "hardware": "-"} + + with patch("netbox_librenms_plugin.utils.find_matching_platform") as mock_platform_match: + mock_platform_match.return_value = {"found": True, "platform": new_platform} + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + assert result["platform_synced"] is False + assert result["all_synced"] is False + + def test_platform_no_match_found_returns_bool(self): + """When find_matching_platform returns no match, platform_synced must be False (not None).""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "ABC123" + existing.platform = MagicMock() # device has a platform set + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + libre_device = {"serial": "ABC123", "os": "ios", "hardware": "-"} + + with patch("netbox_librenms_plugin.utils.find_matching_platform") as mock_platform_match: + mock_platform_match.return_value = {"found": False, "platform": None} + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + # Without bool() cast this would be None; verify it's exactly False (type-stable) + assert result["platform_synced"] is False + assert isinstance(result["platform_synced"], bool) + + def test_platform_synced_no_netbox_platform_returns_bool(self): + """When device has no platform in NetBox and os is non-dash, platform_synced must be bool.""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "ABC123" + existing.platform = None # no platform on device + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + libre_device = {"serial": "ABC123", "os": "eos", "hardware": "-"} + + with patch("netbox_librenms_plugin.utils.find_matching_platform") as mock_platform_match: + mock_platform_match.return_value = {"found": True, "platform": MagicMock()} + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + # None and ... returns None; bool() cast ensures False + assert result["platform_synced"] is False + assert isinstance(result["platform_synced"], bool) + + def test_hardware_no_match_device_type_out_of_sync(self): + """When hardware is present but no device type match found, device_type_synced is False.""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "ABC123" + existing.platform = None + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + libre_device = {"serial": "ABC123", "os": "-", "hardware": "UnknownHardwareXYZ"} + + with patch("netbox_librenms_plugin.utils.match_librenms_hardware_to_device_type") as mock_hw_match: + mock_hw_match.return_value = {"matched": False, "device_type": None} + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + assert result["device_type_synced"] is False + assert result["all_synced"] is False + + +class TestDeviceNamingPreferences: + """Test that validation honours use_sysname and strip_domain user preferences.""" + + def _setup_no_existing(self, mocks): + """Configure mocks so no existing device is found.""" + ( + mock_site_model, + mock_rack, + mock_cluster, + mock_role, + mock_match_type, + mock_find_platform, + mock_find_site, + mock_device, + mock_vm, + ) = mocks + + mock_vm.objects.filter.return_value.first.return_value = None + mock_device.objects.filter.return_value.first.return_value = None + mock_find_site.return_value = { + "found": False, + "site": None, + "match_type": None, + "confidence": 0.0, + } + mock_find_platform.return_value = { + "found": False, + "platform": None, + "match_type": None, + } + mock_match_type.return_value = { + "matched": False, + "device_type": None, + "match_type": None, + } + mock_role.objects.all.return_value = [] + mock_site_model.objects.all.return_value = [] + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_resolved_name_uses_sysname_by_default(self, *mocks): + """Default use_sysname=True uses sysName for resolved_name.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "10.0.0.1", + "sysName": "core-switch", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + assert result["resolved_name"] == "core-switch" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_resolved_name_uses_hostname_when_sysname_disabled(self, *mocks): + """use_sysname=False uses hostname for resolved_name.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "10.0.0.1", + "sysName": "core-switch", + } + result = validate_device_for_import( + device_data, + include_vc_detection=False, + use_sysname=False, + ) + assert result["resolved_name"] == "10.0.0.1" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_resolved_name_strips_domain(self, *mocks): + """strip_domain=True strips the domain suffix.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "switch-01.example.com", + "sysName": "switch-01.example.com", + } + result = validate_device_for_import( + device_data, + include_vc_detection=False, + strip_domain=True, + ) + assert result["resolved_name"] == "switch-01" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_duplicate_detection_uses_resolved_name(self, *mocks): + """Duplicate detection should match against the resolved name, not raw hostname.""" + self._setup_no_existing(mocks) + + # Unpack using same order as _setup_no_existing / @patch decorators (bottom-up) + ( + _mock_site, + _mock_rack, + _mock_cluster, + _mock_role, + _mock_hw, + _mock_platform, + _mock_find_site, + mock_device, + _mock_vm, + ) = mocks + existing = MagicMock() + existing.name = "core-switch" + existing.serial = "" + existing.virtual_chassis = None + existing.vc_position = None + mock_device.objects.filter.return_value.first.side_effect = [None, existing] + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 999, + "hostname": "10.0.0.1", + "sysName": "core-switch", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_device"] == existing + assert result["existing_match_type"] == "hostname" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_backward_compatible_defaults(self, *mocks): + """Calling without naming params produces resolved_name in result.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "switch-01", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + assert "resolved_name" in result + assert result["resolved_name"] == "switch-01" + + +class TestProcessDeviceFilters: + """Tests for process_device_filters and related bulk_import utilities.""" + + def test_show_disabled_filters_integer_disabled_1(self): + """show_disabled=False should exclude devices with disabled==1 (int).""" + from unittest.mock import MagicMock, patch + + devices = [ + {"device_id": 1, "hostname": "a", "disabled": 0, "status": 1}, + {"device_id": 2, "hostname": "b", "disabled": 1, "status": 1}, # disabled in LibreNMS + ] + with ( + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_librenms_devices_for_import", + return_value=(devices, False), + ), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import", + side_effect=lambda d, **kw: { + "resolved_name": d["hostname"], + "is_ready": True, + "can_import": True, + "status": "active", + "existing_device": None, + "import_as_vm": False, + "existing_match_type": None, + }, + ), + patch("netbox_librenms_plugin.import_utils.bulk_import.prefetch_vc_data_for_devices"), + patch("netbox_librenms_plugin.import_utils.bulk_import.cache"), + patch("netbox_librenms_plugin.import_utils.bulk_import.get_cache_metadata_key", return_value="key"), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_validated_device_cache_key", return_value="vkey" + ), + ): + from netbox_librenms_plugin.import_utils.bulk_import import process_device_filters + + api = MagicMock() + api.server_key = "default" + result = process_device_filters( + api, filters={}, vc_detection_enabled=False, clear_cache=False, show_disabled=False + ) + + # Only enabled device (disabled==0) should be processed + assert len(result) == 1 + assert result[0]["hostname"] == "a" + + def test_show_disabled_keeps_unreachable_enabled_device(self): + """show_disabled=False should keep devices that are enabled (disabled==0) even if status==0.""" + from unittest.mock import MagicMock, patch + + devices = [ + {"device_id": 1, "hostname": "a", "disabled": 0, "status": 0}, # down but enabled + {"device_id": 2, "hostname": "b", "disabled": 1, "status": 0}, # down and disabled + ] + with ( + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_librenms_devices_for_import", + return_value=(devices, False), + ), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import", + side_effect=lambda d, **kw: { + "resolved_name": d["hostname"], + "is_ready": True, + "can_import": True, + "status": "active", + "existing_device": None, + "import_as_vm": False, + "existing_match_type": None, + }, + ), + patch("netbox_librenms_plugin.import_utils.bulk_import.prefetch_vc_data_for_devices"), + patch("netbox_librenms_plugin.import_utils.bulk_import.cache"), + patch("netbox_librenms_plugin.import_utils.bulk_import.get_cache_metadata_key", return_value="key"), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_validated_device_cache_key", return_value="vkey" + ), + ): + from netbox_librenms_plugin.import_utils.bulk_import import process_device_filters + + api = MagicMock() + api.server_key = "default" + result = process_device_filters( + api, filters={}, vc_detection_enabled=False, clear_cache=False, show_disabled=False + ) + + # Device a is enabled (disabled==0) and should be kept even though status==0 + assert len(result) == 1 + assert result[0]["hostname"] == "a" + + def test_show_disabled_true_includes_all(self): + """show_disabled=True should include both active and inactive devices.""" + from unittest.mock import MagicMock, patch + + devices = [ + {"device_id": 1, "hostname": "a", "status": 1}, + {"device_id": 2, "hostname": "b", "status": 0}, + ] + with ( + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_librenms_devices_for_import", + return_value=(devices, False), + ), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import", + side_effect=lambda d, **kw: { + "resolved_name": d["hostname"], + "is_ready": True, + "can_import": True, + "status": "active", + "existing_device": None, + "import_as_vm": False, + "existing_match_type": None, + }, + ), + patch("netbox_librenms_plugin.import_utils.bulk_import.prefetch_vc_data_for_devices"), + patch("netbox_librenms_plugin.import_utils.bulk_import.cache"), + patch("netbox_librenms_plugin.import_utils.bulk_import.get_cache_metadata_key", return_value="key"), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_validated_device_cache_key", return_value="vkey" + ), + ): + from netbox_librenms_plugin.import_utils.bulk_import import process_device_filters + + api = MagicMock() + api.server_key = "default" + result = process_device_filters( + api, filters={}, vc_detection_enabled=False, clear_cache=False, show_disabled=True + ) + + assert len(result) == 2 + + def test_empty_return_helper(self): + """_empty_return should return ([], False) when return_cache_status=True, else [].""" + from netbox_librenms_plugin.import_utils.bulk_import import _empty_return + + assert _empty_return(True) == ([], False) + assert _empty_return(False) == [] + + def test_bulk_import_devices_uses_resolved_server_key(self): + """bulk_import_devices_shared should pass api.server_key to import_single_device.""" + from unittest.mock import MagicMock, patch + + with ( + patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") as mock_api_cls, + patch("netbox_librenms_plugin.import_utils.bulk_import.import_single_device") as mock_import, + patch("netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import"), + patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions"), + ): + mock_api = MagicMock() + mock_api.server_key = "resolved-key" + mock_api.get_device_info.return_value = (True, {"device_id": 1, "hostname": "sw"}) + mock_api_cls.return_value = mock_api + mock_import.return_value = {"success": True, "device": MagicMock(), "is_vm": False} + + user = MagicMock() + from netbox_librenms_plugin.import_utils.bulk_import import bulk_import_devices_shared + + bulk_import_devices_shared([1], user=user, server_key=None) + + # The resolved api.server_key ("resolved-key") must be passed, not None + assert mock_import.call_args is not None + assert mock_import.call_args.kwargs.get("server_key") == "resolved-key" + + +class TestVCPositionHandling: + """Test VC position normalization and suggested name generation.""" + + def test_clone_vc_data_position_fallback_is_one_based(self): + """_clone_virtual_chassis_data fallback must be 1-based (idx+1, not idx).""" + from netbox_librenms_plugin.import_utils.virtual_chassis import _clone_virtual_chassis_data + + data = {"is_stack": True, "member_count": 2, "members": [{"serial": "S1"}, {"serial": "S2"}]} + result = _clone_virtual_chassis_data(data) + positions = [m["position"] for m in result["members"]] + # First member: idx=0 β†’ position should be 1, not 0 + assert positions[0] == 1 + assert positions[1] == 2 + + def test_clone_vc_data_preserves_explicit_positions(self): + """_clone_virtual_chassis_data must preserve explicitly set positions.""" + from netbox_librenms_plugin.import_utils.virtual_chassis import _clone_virtual_chassis_data + + data = { + "is_stack": True, + "member_count": 2, + "members": [{"serial": "S1", "position": 3}, {"serial": "S2", "position": 5}], + } + result = _clone_virtual_chassis_data(data) + assert result["members"][0]["position"] == 3 + assert result["members"][1]["position"] == 5 + + def test_clone_vc_data_bad_position_falls_back_to_one_based(self): + """_clone_virtual_chassis_data falls back to idx+1 for non-int position.""" + from netbox_librenms_plugin.import_utils.virtual_chassis import _clone_virtual_chassis_data + + data = { + "is_stack": True, + "member_count": 2, + "members": [{"serial": "S1", "position": "bad"}, {"serial": "S2", "position": None}], + } + result = _clone_virtual_chassis_data(data) + # idx=0 β†’ fallback 1, idx=1 β†’ fallback 2 + assert result["members"][0]["position"] == 1 + assert result["members"][1]["position"] == 2 + + def test_suggested_name_uses_position_directly(self): + """Suggested name generation must use position directly (not position+1). + + This test verifies that _generate_vc_member_name is called with the + already-1-based position value, not position+1. + """ + from netbox_librenms_plugin.import_utils.virtual_chassis import _generate_vc_member_name + + # position=1 should produce name with "1", not "2" + name = _generate_vc_member_name("switch-1", 1, pattern="-M{position}") + assert name == "switch-1-M1", f"Expected 'switch-1-M1', got '{name}'" + + # position=2 should produce "2", not "3" + name = _generate_vc_member_name("switch-1", 2, pattern="-M{position}") + assert name == "switch-1-M2", f"Expected 'switch-1-M2', got '{name}'" + + +class TestBulkImportCancellation: + """Test that bulk_import_devices_shared respects RQ and DB cancellation.""" + + def _run_bulk_import(self, mock_rq_job=None, db_status="running", device_ids=None): + """Helper: run bulk_import with provided mocks, return import call count.""" + from unittest.mock import MagicMock, patch + + if device_ids is None: + device_ids = [1, 2, 3, 4, 5, 6] + + job = MagicMock() + job.job.job_id = "test-uuid" + job_status = MagicMock() + job_status.value = db_status + job.job.status = job_status + job.logger = MagicMock() + + with ( + patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") as mock_api_cls, + patch("netbox_librenms_plugin.import_utils.bulk_import.import_single_device") as mock_import, + patch("netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import"), + patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions"), + # Inline imports in the loop use django_rq.get_queue / rq.job.Job directly + patch("django_rq.get_queue") as mock_get_queue, + patch("rq.job.Job") as mock_rqjob_cls, + ): + mock_api = MagicMock() + mock_api.server_key = "default" + mock_api.get_device_info.return_value = (True, {"device_id": 1, "hostname": "sw"}) + mock_api_cls.return_value = mock_api + mock_import.return_value = {"success": True, "device": MagicMock(), "is_vm": False} + + if mock_rq_job is not None: + mock_conn = MagicMock() + mock_queue = MagicMock() + mock_queue.connection = mock_conn + mock_get_queue.return_value = mock_queue + mock_rqjob_cls.fetch.return_value = mock_rq_job + else: + # Simulate RQ unavailable β€” get_queue raises, triggers DB fallback + mock_get_queue.side_effect = Exception("RQ unavailable") + + from netbox_librenms_plugin.import_utils.bulk_import import bulk_import_devices_shared + + bulk_import_devices_shared(device_ids, user=MagicMock(), server_key=None, job=job) + + return mock_import.call_count + + def test_rq_stopped_cancels_import_loop(self): + """When RQ job is_stopped, import loop should break early.""" + rq_job = MagicMock() + rq_job.is_stopped = True + rq_job.is_failed = False + rq_job.get_status.return_value = "stopped" + + # With 6 devices and RQ stopped on first check (idx=1), at most 1 device processed + count = self._run_bulk_import(mock_rq_job=rq_job, device_ids=[1, 2, 3, 4, 5, 6]) + assert count == 0 # break before first import + + def test_rq_failed_cancels_import_loop(self): + """When RQ job is_failed, import loop should break early.""" + rq_job = MagicMock() + rq_job.is_stopped = False + rq_job.is_failed = True + rq_job.get_status.return_value = "failed" + + count = self._run_bulk_import(mock_rq_job=rq_job, device_ids=[1, 2, 3, 4, 5, 6]) + assert count == 0 + + def test_rq_unavailable_falls_back_to_db_check(self): + """When RQ is unavailable, DB status check is used as fallback.""" + # mock_rq_job=None triggers the side_effect=Exception path + count = self._run_bulk_import(mock_rq_job=None, db_status="failed", device_ids=[1]) + # With DB status "failed", import should not run + assert count == 0 + + def test_db_errored_status_also_terminates_loop(self): + """When DB job status is 'errored', import loop should terminate early.""" + count = self._run_bulk_import(mock_rq_job=None, db_status="errored", device_ids=[1, 2, 3]) + assert count == 0 + + def test_healthy_job_runs_all_devices(self): + """When job is healthy, all devices should be imported.""" + rq_job = MagicMock() + rq_job.is_stopped = False + rq_job.is_failed = False + rq_job.get_status.return_value = "started" + + count = self._run_bulk_import(mock_rq_job=rq_job, device_ids=[1, 2, 3]) + assert count == 3 + + +class TestBuildIdServerInfo: + """Test DeviceValidationDetailsView._build_id_server_info method.""" + + def _make_device(self, librenms_id_value): + device = MagicMock() + device.custom_field_data = {"librenms_id": librenms_id_value} + return device + + def test_returns_none_for_legacy_int(self): + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + device = self._make_device(42) + result = DeviceValidationDetailsView._build_id_server_info(device) + assert result is None + + def test_returns_none_for_missing_cf(self): + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + device = self._make_device(None) + result = DeviceValidationDetailsView._build_id_server_info(device) + assert result is None + + def test_single_server_resolves_display_name(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + device = self._make_device({"production": 42}) + plugins_cfg = { + "netbox_librenms_plugin": { + "servers": { + "production": {"display_name": "Production LibreNMS", "librenms_url": "https://prod.example.com"}, + } + } + } + with patch("django.conf.settings") as mock_settings: + mock_settings.PLUGINS_CONFIG = plugins_cfg + result = DeviceValidationDetailsView._build_id_server_info(device) + + assert result is not None + assert len(result) == 1 + assert result[0]["server_key"] == "production" + assert result[0]["display_name"] == "Production LibreNMS" + assert result[0]["device_id"] == 42 + + def test_unconfigured_server_uses_key_as_display_name(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + device = self._make_device({"deleted-server": 77}) + plugins_cfg = {"netbox_librenms_plugin": {"servers": {}}} + with patch("django.conf.settings") as mock_settings: + mock_settings.PLUGINS_CONFIG = plugins_cfg + result = DeviceValidationDetailsView._build_id_server_info(device) + + assert result is not None + assert result[0]["display_name"] == "deleted-server" + + def test_empty_dict_returns_none(self): + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + device = self._make_device({}) + result = DeviceValidationDetailsView._build_id_server_info(device) + assert result is None + + +# --------------------------------------------------------------------------- +# Tests for _refresh_existing_device sys_name fallback fix +# --------------------------------------------------------------------------- + + +class TestRefreshExistingDeviceSysNameFallback: + """Test that _refresh_existing_device tries sys_name even when hostname is empty.""" + + def test_sysname_used_when_hostname_empty(self): + """When hostname is empty but sys_name matches, the device is found in validation.""" + from unittest.mock import MagicMock, patch + + from netbox_librenms_plugin.import_utils.bulk_import import _refresh_existing_device + + mock_device = MagicMock() + mock_device.pk = 99 + mock_device.name = "router-01" + mock_device.custom_field_data = {"librenms_id": None} + + libre_device = { + "device_id": 55, + "hostname": "", # empty hostname + "sysName": "router-01", + "serial": "SN-MATCH", + } + validation = { + "existing_device": None, + "existing_vm": None, + "import_as_vm": False, + "is_ready": False, + "can_import": False, + } + + # sys_name lookup: filter(name__iexact="router-01") returns mock_device + # hostname lookup: filter(name__iexact="") returns None + def make_qs(return_val): + qs = MagicMock() + qs.first.return_value = return_val + return qs + + with patch("netbox_librenms_plugin.import_utils.bulk_import.find_by_librenms_id", return_value=None): + import dcim.models as dcim_models + import virtualization.models as virt_models + + with ( + patch.object( + dcim_models.Device.objects, + "filter", + side_effect=lambda **kw: make_qs(mock_device if kw.get("name__iexact") == "router-01" else None), + ), + patch.object(virt_models.VirtualMachine.objects, "filter", return_value=make_qs(None)), + ): + _refresh_existing_device(validation, libre_device=libre_device, server_key="default") + + assert validation["existing_device"] is mock_device + + def test_hostname_lookup_succeeds_without_sysname(self): + """When hostname is non-empty and matches, validation is updated correctly.""" + from unittest.mock import MagicMock, patch + + from netbox_librenms_plugin.import_utils.bulk_import import _refresh_existing_device + + mock_device = MagicMock() + mock_device.pk = 10 + mock_device.name = "sw-01" + mock_device.custom_field_data = {"librenms_id": None} + + libre_device = { + "device_id": 10, + "hostname": "sw-01", + "sysName": "sw-01-sysname", + "serial": "", + } + validation = { + "existing_device": None, + "existing_vm": None, + "import_as_vm": False, + "is_ready": False, + "can_import": False, + } + + def make_qs(return_val): + qs = MagicMock() + qs.first.return_value = return_val + return qs + + with patch("netbox_librenms_plugin.import_utils.bulk_import.find_by_librenms_id", return_value=None): + import dcim.models as dcim_models + import virtualization.models as virt_models + + with ( + patch.object( + dcim_models.Device.objects, + "filter", + side_effect=lambda **kw: make_qs(mock_device if kw.get("name__iexact") == "sw-01" else None), + ), + patch.object(virt_models.VirtualMachine.objects, "filter", return_value=make_qs(None)), + ): + _refresh_existing_device(validation, libre_device=libre_device, server_key="default") + + assert validation["existing_device"] is mock_device + + +# --------------------------------------------------------------------------- +# Tests for _get_hostname_for_action helper +# --------------------------------------------------------------------------- + + +class TestGetHostnameForAction: + """Test _get_hostname_for_action helper in actions.py.""" + + def test_returns_resolved_name_when_set(self): + from unittest.mock import MagicMock + + from netbox_librenms_plugin.views.imports.actions import _get_hostname_for_action + + request = MagicMock() + validation = {"resolved_name": "cached-name"} + libre_device = {"hostname": "raw-hostname", "sysName": "raw-sysname"} + + result = _get_hostname_for_action(request, validation, libre_device) + assert result == "cached-name" + + def test_falls_back_to_determine_device_name(self): + from unittest.mock import MagicMock, patch + + from netbox_librenms_plugin.views.imports.actions import _get_hostname_for_action + + request = MagicMock() + validation = {} # no resolved_name + libre_device = {"hostname": "host.example.com", "sysName": "host"} + + with patch("netbox_librenms_plugin.views.imports.actions._resolve_naming_preferences") as mock_prefs: + mock_prefs.return_value = (False, False) # use_sysname=False, strip_domain=False + with patch("netbox_librenms_plugin.views.imports.actions._determine_device_name") as mock_name: + mock_name.return_value = "host.example.com" + result = _get_hostname_for_action(request, validation, libre_device) + + assert result == "host.example.com" + mock_prefs.assert_called_once_with(request) + mock_name.assert_called_once() + + +# --------------------------------------------------------------------------- +# Tests for _resolve_naming_preferences underscore-variant key support +# --------------------------------------------------------------------------- + + +class TestResolveNamingPreferencesKeys: + """Test that _resolve_naming_preferences handles both hyphenated and underscored keys.""" + + def _make_request(self, post=None, get=None): + from unittest.mock import MagicMock + + request = MagicMock() + request.POST = post or {} + request.GET = get or {} + request.user = MagicMock() + return request + + def test_hyphenated_post_key_use_sysname(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import _resolve_naming_preferences + + request = self._make_request(post={"use-sysname-toggle": "on", "strip-domain-toggle": "off"}) + with patch("netbox_librenms_plugin.views.imports.actions.get_user_pref", return_value=None): + use_sysname, strip_domain = _resolve_naming_preferences(request) + assert use_sysname is True + assert strip_domain is False + + def test_underscored_post_key_use_sysname(self): + """Underscore variant 'use_sysname-toggle' should also be recognised.""" + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import _resolve_naming_preferences + + request = self._make_request(post={"use_sysname-toggle": "on", "strip_domain-toggle": "on"}) + with patch("netbox_librenms_plugin.views.imports.actions.get_user_pref", return_value=None): + use_sysname, strip_domain = _resolve_naming_preferences(request) + assert use_sysname is True + assert strip_domain is True + + def test_get_key_used_when_not_in_post(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import _resolve_naming_preferences + + request = self._make_request(get={"use-sysname-toggle": "off", "strip-domain-toggle": "on"}) + with patch("netbox_librenms_plugin.views.imports.actions.get_user_pref", return_value=None): + use_sysname, strip_domain = _resolve_naming_preferences(request) + assert use_sysname is False + assert strip_domain is True + + def test_user_pref_used_when_no_toggle_in_request(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import _resolve_naming_preferences + + request = self._make_request() + with patch("netbox_librenms_plugin.views.imports.actions.get_user_pref") as mock_pref: + mock_pref.side_effect = lambda req, key: (False if "use_sysname" in key else True) + use_sysname, strip_domain = _resolve_naming_preferences(request) + assert use_sysname is False + assert strip_domain is True + + def test_post_takes_precedence_over_user_pref(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import _resolve_naming_preferences + + request = self._make_request(post={"use-sysname-toggle": "off"}) + # user_pref would say True β€” POST should win + with patch("netbox_librenms_plugin.views.imports.actions.get_user_pref", return_value=True): + use_sysname, _ = _resolve_naming_preferences(request) + assert use_sysname is False + + def test_truthy_string_true_value(self): + """'true' and '1' (in addition to 'on') should be treated as True.""" + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import _resolve_naming_preferences + + for truthy_val in ("true", "True", "TRUE", "1"): + request = self._make_request(post={"use-sysname-toggle": truthy_val, "strip-domain-toggle": "off"}) + with patch("netbox_librenms_plugin.views.imports.actions.get_user_pref", return_value=None): + use_sysname, _ = _resolve_naming_preferences(request) + assert use_sysname is True, f"Expected True for value {truthy_val!r}" + + def test_falsy_string_false_value(self): + """Unrecognised strings should be treated as False.""" + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import _resolve_naming_preferences + + for falsy_val in ("off", "false", "0", "", "no"): + request = self._make_request(post={"use-sysname-toggle": falsy_val, "strip-domain-toggle": "off"}) + with patch("netbox_librenms_plugin.views.imports.actions.get_user_pref", return_value=None): + use_sysname, _ = _resolve_naming_preferences(request) + assert use_sysname is False, f"Expected False for value {falsy_val!r}" + + +# --------------------------------------------------------------------------- +# Tests for vc_domain stack dedup key fix +# --------------------------------------------------------------------------- + + +class TestVCDomainStackDedup: + """Test that bulk_import_devices_shared deduplicates VC creation by member serials.""" + + def test_vc_domain_uses_member_serials(self): + """vc_domain for two stack members with the same serials should be identical.""" + # The logic lives inline; test the produced key directly from vc_data + members = [ + {"serial": "SN100", "position": 1}, + {"serial": "SN200", "position": 2}, + ] + member_serials = sorted(m.get("serial") for m in members if m.get("serial")) + vc_domain = f"librenms-stack-{','.join(member_serials)}" + + # Same members from a different device's perspective should produce the same key + assert vc_domain == "librenms-stack-SN100,SN200" + + def test_vc_domain_fallback_to_device_id_when_no_serials(self): + """When no member serials are available, device_id is used as fallback.""" + members = [ + {"position": 1}, + {"position": 2}, + ] + member_serials = sorted(m.get("serial") for m in members if m.get("serial")) + device_id = 42 + vc_domain = f"librenms-stack-{','.join(member_serials)}" if member_serials else f"librenms-{device_id}" + assert vc_domain == "librenms-42" + + def test_different_stacks_produce_different_keys(self): + """Two stacks with different serials produce distinct dedup keys.""" + members_a = [{"serial": "SN-A1"}, {"serial": "SN-A2"}] + members_b = [{"serial": "SN-B1"}, {"serial": "SN-B2"}] + key_a = f"librenms-stack-{','.join(sorted(m['serial'] for m in members_a))}" + key_b = f"librenms-stack-{','.join(sorted(m['serial'] for m in members_b))}" + assert key_a != key_b diff --git a/netbox_librenms_plugin/tests/test_init.py b/netbox_librenms_plugin/tests/test_init.py new file mode 100644 index 0000000000..426f6736c0 --- /dev/null +++ b/netbox_librenms_plugin/tests/test_init.py @@ -0,0 +1,174 @@ +"""Tests for netbox_librenms_plugin.__init__ module. + +Covers the _ensure_librenms_id_custom_field post_migrate signal handler. +""" + +from unittest.mock import MagicMock, patch + + +# ============================================================================= +# TestEnsureLibreNMSIdCustomField - 6 tests +# ============================================================================= + + +class TestEnsureLibreNMSIdCustomField: + """Test _ensure_librenms_id_custom_field signal handler.""" + + def setup_method(self): + """Reset the _executed flag before each test for consistent isolation.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + _ensure_librenms_id_custom_field._executed = False + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_creates_custom_field_when_missing( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """Custom field is created with correct defaults when it does not exist.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [] + MockCustomField.objects.get_or_create.return_value = (mock_cf, True) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + with patch("logging.getLogger") as mock_get_logger: + _ensure_librenms_id_custom_field(sender=None) + + MockCustomField.objects.get_or_create.assert_called_once_with( + name="librenms_id", + defaults={ + "type": "json", + "label": "LibreNMS ID", + "description": "LibreNMS Device ID for synchronization (auto-created by plugin)", + "required": False, + "ui_visible": "if-set", + "ui_editable": "yes", + "is_cloneable": False, + }, + ) + + # Should have added content types for all 4 models + assert mock_cf.object_types.add.call_count == 4 + + # Should log when created + mock_get_logger.assert_called_with("netbox_librenms_plugin") + mock_get_logger.return_value.info.assert_called_once() + + def test_skips_when_already_executed(self): + """Handler is a no-op on second invocation (per-migrate dedup).""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + _ensure_librenms_id_custom_field._executed = True + + with patch("extras.models.CustomField") as MockCustomField: + _ensure_librenms_id_custom_field(sender=None) + MockCustomField.objects.get_or_create.assert_not_called() + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_existing_field_not_recreated( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """When custom field already exists, it is not recreated but types are checked.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2, 3, 4] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + _ensure_librenms_id_custom_field(sender=None) + + # All pks already present, no types should be added + mock_cf.object_types.add.assert_not_called() + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_adds_missing_content_types( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """When some content types are missing, only those are added.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + ct_existing = MagicMock() + ct_existing.pk = 1 + ct_new = MagicMock() + ct_new.pk = 99 + MockContentType.objects.get_for_model.side_effect = [ct_existing, ct_existing, ct_new, ct_new] + + _ensure_librenms_id_custom_field(sender=None) + + assert mock_cf.object_types.add.call_count == 2 + mock_cf.object_types.add.assert_any_call(ct_new) + + @patch("extras.models.CustomField") + def test_exception_does_not_propagate(self, MockCustomField): + """Exceptions during custom field creation are caught and logged.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + MockCustomField.objects.get_or_create.side_effect = Exception("DB not ready") + + with patch("logging.getLogger") as mock_get_logger: + # Should not raise + _ensure_librenms_id_custom_field(sender=None) + + # Verify the exception was logged + logger_instance = mock_get_logger.return_value + logger_instance.exception.assert_called_once() + call_args = logger_instance.exception.call_args + assert "librenms_id" in call_args[0][0] + + # On failure, _executed must NOT be set β€” failed attempts should allow retry + assert not getattr(_ensure_librenms_id_custom_field, "_executed", False) + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_no_log_when_field_already_exists( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """No log message when the custom field already existed.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2, 3, 4] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + with patch("logging.getLogger") as mock_get_logger: + _ensure_librenms_id_custom_field(sender=None) + # When the field already exists (created=False), the info log should + # not be emitted. We verify via the logger instance rather than + # asserting getLogger was never called, which is fragile. + logger_instance = mock_get_logger.return_value + logger_instance.info.assert_not_called() diff --git a/netbox_librenms_plugin/tests/test_librenms_api.py b/netbox_librenms_plugin/tests/test_librenms_api.py index 5ce115f340..2db833dd1c 100644 --- a/netbox_librenms_plugin/tests/test_librenms_api.py +++ b/netbox_librenms_plugin/tests/test_librenms_api.py @@ -643,7 +643,7 @@ def test_add_device_snmpv1_success(self, mock_post, mock_librenms_config): @patch("netbox_librenms_plugin.librenms_api.requests.post") def test_add_device_duplicate_error(self, mock_post, mock_librenms_config): """Verify duplicate device handling.""" - mock_post.return_value.status_code = 500 + mock_post.return_value.status_code = 200 mock_post.return_value.json.return_value = { "status": "error", "message": "Device already exists", diff --git a/netbox_librenms_plugin/tests/test_permissions.py b/netbox_librenms_plugin/tests/test_permissions.py index d366965ead..bb6ea62caf 100644 --- a/netbox_librenms_plugin/tests/test_permissions.py +++ b/netbox_librenms_plugin/tests/test_permissions.py @@ -633,8 +633,8 @@ class TestView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin): class TestBulkImportPermissions: """Tests for permission checks in bulk import functions.""" - @patch("netbox_librenms_plugin.import_utils.require_permissions") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions") + @patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") def test_bulk_import_devices_checks_permissions(self, mock_api_class, mock_require): """bulk_import_devices_shared calls require_permissions.""" from netbox_librenms_plugin.import_utils import bulk_import_devices_shared @@ -658,8 +658,8 @@ def test_bulk_import_devices_checks_permissions(self, mock_api_class, mock_requi assert "dcim.add_device" in call_args[0][1] assert "dcim.add_interface" in call_args[0][1] - @patch("netbox_librenms_plugin.import_utils.require_permissions") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions") + @patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") def test_bulk_import_devices_extracts_user_from_job(self, mock_api_class, mock_require): """bulk_import_devices_shared extracts user from job if not provided.""" from netbox_librenms_plugin.import_utils import bulk_import_devices_shared @@ -682,7 +682,7 @@ def test_bulk_import_devices_extracts_user_from_job(self, mock_api_class, mock_r call_args = mock_require.call_args assert job_user == call_args[0][0] - @patch("netbox_librenms_plugin.import_utils.require_permissions") + @patch("netbox_librenms_plugin.import_utils.vm_operations.require_permissions") def test_bulk_import_vms_checks_permissions(self, mock_require): """bulk_import_vms calls require_permissions.""" from netbox_librenms_plugin.import_utils import bulk_import_vms @@ -703,7 +703,7 @@ def test_bulk_import_vms_checks_permissions(self, mock_require): assert user == call_args[0][0] assert "virtualization.add_virtualmachine" in call_args[0][1] - @patch("netbox_librenms_plugin.import_utils.require_permissions") + @patch("netbox_librenms_plugin.import_utils.vm_operations.require_permissions") def test_bulk_import_vms_extracts_user_from_job(self, mock_require): """bulk_import_vms extracts user from job if not provided.""" from netbox_librenms_plugin.import_utils import bulk_import_vms @@ -729,7 +729,7 @@ def test_bulk_import_vms_extracts_user_from_job(self, mock_require): class TestBulkImportPermissionDenied: """Tests for permission denied behavior in bulk import.""" - @patch("netbox_librenms_plugin.import_utils.check_user_permissions") + @patch("netbox_librenms_plugin.import_utils.permissions.check_user_permissions") def test_bulk_import_devices_raises_on_missing_permissions(self, mock_check): """bulk_import_devices_shared raises PermissionDenied when permissions missing.""" import pytest @@ -748,7 +748,7 @@ def test_bulk_import_devices_raises_on_missing_permissions(self, mock_check): server_key="default", ) - @patch("netbox_librenms_plugin.import_utils.check_user_permissions") + @patch("netbox_librenms_plugin.import_utils.permissions.check_user_permissions") def test_bulk_import_vms_raises_on_missing_permissions(self, mock_check): """bulk_import_vms raises PermissionDenied when permissions missing.""" import pytest @@ -873,8 +873,8 @@ def test_htmx_rejects_external_referrer(self): class TestBulkImportVCPermission: """Tests that bulk import checks virtualchassis permission.""" - @patch("netbox_librenms_plugin.import_utils.require_permissions") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions") + @patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") def test_bulk_import_devices_checks_vc_permission(self, mock_api_class, mock_require): """bulk_import_devices_shared includes dcim.add_virtualchassis in required perms.""" from netbox_librenms_plugin.import_utils import bulk_import_devices_shared @@ -951,3 +951,130 @@ def test_delete_interfaces_invalid_type_raises_404(self): view = DeleteNetBoxInterfacesView() with pytest.raises(Http404): view.get_required_permissions_for_object_type("invalid") + + +# --------------------------------------------------------------------------- +# Tests for RemoveServerMappingView error handling (device_fields.py) +# --------------------------------------------------------------------------- + + +class TestRemoveServerMappingViewErrorHandling: + """Test RemoveServerMappingView handles full_clean/save failures gracefully.""" + + def _make_view(self, server_key, post_extra=None): + """Return a (view, request) pair with permissions satisfied.""" + from unittest.mock import MagicMock + + from netbox_librenms_plugin.views.sync.device_fields import RemoveServerMappingView + + request = MagicMock() + request.POST = {"server_key": server_key, **(post_extra or {})} + request.user = MagicMock() + request.user.has_perm.return_value = True + + view = RemoveServerMappingView() + view.request = request # required by mixin's has_write_permission + return view, request + + def test_validation_error_returns_error_message(self): + """ValidationError from full_clean leads to error message, not 500.""" + from unittest.mock import MagicMock, patch + + from django.core.exceptions import ValidationError + + view, request = self._make_view(server_key="orphan-server") + + mock_device = MagicMock() + mock_device.custom_field_data = {"librenms_id": {"orphan-server": 99}} + + mock_locked = MagicMock() + mock_locked.custom_field_data = {"librenms_id": {"orphan-server": 99}} + mock_locked.full_clean.side_effect = ValidationError("CF validation failed") + + plugins_cfg = {"netbox_librenms_plugin": {"servers": {}}} # orphan-server NOT configured + + with ( + patch("netbox_librenms_plugin.views.sync.device_fields.get_object_or_404", return_value=mock_device), + patch("netbox_librenms_plugin.views.sync.device_fields.Device") as mock_Device_cls, + patch("django.conf.settings") as mock_settings, + patch("netbox_librenms_plugin.views.sync.device_fields.messages") as mock_messages, + patch("netbox_librenms_plugin.views.sync.device_fields.redirect"), + patch("netbox_librenms_plugin.views.sync.device_fields.transaction") as mock_tx, + ): + mock_settings.PLUGINS_CONFIG = plugins_cfg + mock_Device_cls.objects.select_for_update.return_value.get.return_value = mock_locked + + # Make transaction.atomic() a no-op context manager + mock_tx.atomic.return_value.__enter__ = lambda s: None + mock_tx.atomic.return_value.__exit__ = lambda s, *a: None + mock_tx.set_rollback = MagicMock() + + view.post(request, pk=1) + + mock_messages.error.assert_called_once() + error_args = mock_messages.error.call_args[0] + assert "Validation error" in str(error_args[1]) or "CF validation failed" in str(error_args[1]) + + def test_configured_server_refused(self): + """Configured server mapping cannot be removed β€” error message shown.""" + from unittest.mock import MagicMock, patch + + view, request = self._make_view(server_key="active-server") + mock_device = MagicMock() + mock_device.custom_field_data = {"librenms_id": {"active-server": 5}} + + plugins_cfg = {"netbox_librenms_plugin": {"servers": {"active-server": {"librenms_url": "http://x"}}}} + + with ( + patch("netbox_librenms_plugin.views.sync.device_fields.get_object_or_404", return_value=mock_device), + patch("django.conf.settings") as mock_settings, + patch("netbox_librenms_plugin.views.sync.device_fields.messages") as mock_messages, + patch("netbox_librenms_plugin.views.sync.device_fields.redirect"), + ): + mock_settings.PLUGINS_CONFIG = plugins_cfg + view.post(request, pk=1) + + mock_messages.error.assert_called_once() + assert "Cannot remove" in mock_messages.error.call_args[0][1] + + def test_successful_removal_mutates_and_saves(self): + """Successful removal deletes the key from custom_field_data and saves the device.""" + from unittest.mock import MagicMock, patch + + view, request = self._make_view(server_key="orphan-server") + + mock_device = MagicMock() + mock_device.custom_field_data = {"librenms_id": {"orphan-server": 42, "other-server": 7}} + + mock_locked = MagicMock() + mock_locked.custom_field_data = {"librenms_id": {"orphan-server": 42, "other-server": 7}} + mock_locked.full_clean = MagicMock() + mock_locked.save = MagicMock() + + plugins_cfg = {"netbox_librenms_plugin": {"servers": {}}} # orphan-server NOT configured + + with ( + patch("netbox_librenms_plugin.views.sync.device_fields.get_object_or_404", return_value=mock_device), + patch("netbox_librenms_plugin.views.sync.device_fields.Device") as mock_Device_cls, + patch("django.conf.settings") as mock_settings, + patch("netbox_librenms_plugin.views.sync.device_fields.messages") as mock_messages, + patch("netbox_librenms_plugin.views.sync.device_fields.redirect"), + patch("netbox_librenms_plugin.views.sync.device_fields.transaction") as mock_tx, + ): + mock_settings.PLUGINS_CONFIG = plugins_cfg + mock_Device_cls.objects.select_for_update.return_value.get.return_value = mock_locked + + mock_tx.atomic.return_value.__enter__ = lambda s: None + mock_tx.atomic.return_value.__exit__ = lambda s, *a: None + mock_tx.set_rollback = MagicMock() + + view.post(request, pk=1) + + # The "orphan-server" key should have been removed and the device saved. + # Assert the exact shape of custom_field_data so misspelled keys are caught. + assert mock_locked.custom_field_data == {"librenms_id": {"other-server": 7}} + remaining = mock_locked.custom_field_data["librenms_id"] + assert "orphan-server" not in remaining + assert remaining.get("other-server") == 7 # sibling key preserved + mock_locked.save.assert_called_once() + mock_messages.success.assert_called_once() diff --git a/netbox_librenms_plugin/tests/test_sync_view_mismatch.py b/netbox_librenms_plugin/tests/test_sync_view_mismatch.py index e59ab6909c..e9a04d2997 100644 --- a/netbox_librenms_plugin/tests/test_sync_view_mismatch.py +++ b/netbox_librenms_plugin/tests/test_sync_view_mismatch.py @@ -318,3 +318,112 @@ def test_vc_pattern_no_match_leaves_name(self, mock_settings_qs, mock_hw): result = view.get_librenms_device_info(obj) assert result["mismatched_device"] is True + + +# --------------------------------------------------------------------------- +# Tests for _build_all_server_mappings +# --------------------------------------------------------------------------- + + +class TestBuildAllServerMappings: + """Tests for BaseLibreNMSSyncView._build_all_server_mappings.""" + + def _make_obj(self, cf_librenms_id): + obj = MagicMock() + obj.custom_field_data = {"librenms_id": cf_librenms_id} + return obj + + def test_returns_none_for_legacy_int(self): + from netbox_librenms_plugin.views.base.librenms_sync_view import BaseLibreNMSSyncView + + obj = self._make_obj(42) + result = BaseLibreNMSSyncView._build_all_server_mappings(obj, "production") + assert result is None + + def test_returns_none_for_missing_cf(self): + from netbox_librenms_plugin.views.base.librenms_sync_view import BaseLibreNMSSyncView + + obj = self._make_obj(None) + result = BaseLibreNMSSyncView._build_all_server_mappings(obj, "production") + assert result is None + + def test_single_configured_server(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.base.librenms_sync_view import BaseLibreNMSSyncView + + obj = self._make_obj({"production": 42}) + plugins_cfg = { + "netbox_librenms_plugin": { + "servers": { + "production": { + "display_name": "Production LibreNMS", + "librenms_url": "https://librenms.example.com", + }, + } + } + } + with patch("netbox_librenms_plugin.views.base.librenms_sync_view.django_settings") as mock_settings: + mock_settings.PLUGINS_CONFIG = plugins_cfg + result = BaseLibreNMSSyncView._build_all_server_mappings(obj, "production") + + assert result is not None + assert len(result) == 1 + entry = result[0] + assert entry["server_key"] == "production" + assert entry["device_id"] == 42 + assert entry["display_name"] == "Production LibreNMS" + assert entry["is_configured"] is True + assert entry["is_active"] is True + assert entry["device_url"] == "https://librenms.example.com/device/device=42/" + + def test_orphaned_server_is_not_configured(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.base.librenms_sync_view import BaseLibreNMSSyncView + + obj = self._make_obj({"deleted-server": 77}) + plugins_cfg = {"netbox_librenms_plugin": {"servers": {}}} + with patch("netbox_librenms_plugin.views.base.librenms_sync_view.django_settings") as mock_settings: + mock_settings.PLUGINS_CONFIG = plugins_cfg + result = BaseLibreNMSSyncView._build_all_server_mappings(obj, "production") + + assert result is not None + assert len(result) == 1 + entry = result[0] + assert entry["server_key"] == "deleted-server" + assert entry["device_id"] == 77 + assert entry["is_configured"] is False + assert entry["is_active"] is False + assert entry["device_url"] is None + + def test_multiple_servers_sorted_active_first(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.base.librenms_sync_view import BaseLibreNMSSyncView + + obj = self._make_obj({"mock-dev": 99, "production": 42, "old-server": 11}) + plugins_cfg = { + "netbox_librenms_plugin": { + "servers": { + "production": {"display_name": "Production", "librenms_url": "https://prod.example.com"}, + "mock-dev": {"display_name": "Mock", "librenms_url": "http://mock.example.com"}, + } + } + } + with patch("netbox_librenms_plugin.views.base.librenms_sync_view.django_settings") as mock_settings: + mock_settings.PLUGINS_CONFIG = plugins_cfg + result = BaseLibreNMSSyncView._build_all_server_mappings(obj, "production") + + assert result is not None + assert len(result) == 3 + # Active (production) first + assert result[0]["server_key"] == "production" + assert result[0]["is_active"] is True + # Configured (mock-dev) second + assert result[1]["server_key"] == "mock-dev" + assert result[1]["is_configured"] is True + assert result[1]["is_active"] is False + # Orphaned last + assert result[2]["server_key"] == "old-server" + assert result[2]["is_configured"] is False diff --git a/netbox_librenms_plugin/tests/test_utils.py b/netbox_librenms_plugin/tests/test_utils.py index 96065ab760..d98ca28b9c 100644 --- a/netbox_librenms_plugin/tests/test_utils.py +++ b/netbox_librenms_plugin/tests/test_utils.py @@ -15,9 +15,12 @@ class TestDeviceTypeMatching: """Test device type matching logic.""" + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") @patch("dcim.models.DeviceType") - def test_match_device_type_exact_match_by_part_number(self, mock_device_type): + def test_match_device_type_exact_match_by_part_number(self, mock_device_type, mock_mapping): """Exact part_number string should match.""" + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.side_effect = mock_mapping.DoesNotExist mock_dt = MagicMock(id=1, model="C9300-48P") mock_device_type.objects.get.return_value = mock_dt @@ -29,9 +32,12 @@ def test_match_device_type_exact_match_by_part_number(self, mock_device_type): assert result["device_type"] == mock_dt assert result["match_type"] == "exact" + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") @patch("dcim.models.DeviceType") - def test_match_device_type_exact_match_by_model(self, mock_device_type): + def test_match_device_type_exact_match_by_model(self, mock_device_type, mock_mapping): """Exact model string should match when part_number fails.""" + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.side_effect = mock_mapping.DoesNotExist mock_dt = MagicMock(id=1, model="WS-C3750X-48P") # Part number lookup fails, model lookup succeeds mock_device_type.DoesNotExist = Exception @@ -48,9 +54,12 @@ def test_match_device_type_exact_match_by_model(self, mock_device_type): assert result["device_type"] == mock_dt assert result["match_type"] == "exact" + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") @patch("dcim.models.DeviceType") - def test_match_device_type_not_found(self, mock_device_type): + def test_match_device_type_not_found(self, mock_device_type, mock_mapping): """Returns None when no match found.""" + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.side_effect = mock_mapping.DoesNotExist mock_device_type.DoesNotExist = Exception mock_device_type.objects.get.side_effect = mock_device_type.DoesNotExist @@ -62,6 +71,22 @@ def test_match_device_type_not_found(self, mock_device_type): assert result["device_type"] is None assert result["match_type"] is None + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") + def test_match_device_type_mapping_match(self, mock_mapping): + """DeviceTypeMapping entry should be used before part_number/model fallback.""" + mock_dt = MagicMock(id=1, model="MX480") + mock_mapping_obj = MagicMock(netbox_device_type=mock_dt) + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.return_value = mock_mapping_obj + + from netbox_librenms_plugin.utils import match_librenms_hardware_to_device_type + + result = match_librenms_hardware_to_device_type("Juniper MX480 Internet Backbone Router") + + assert result["matched"] is True + assert result["device_type"] == mock_dt + assert result["match_type"] == "mapping" + def test_match_device_type_empty_hardware(self): """Empty string returns None.""" from netbox_librenms_plugin.utils import match_librenms_hardware_to_device_type diff --git a/netbox_librenms_plugin/urls.py b/netbox_librenms_plugin/urls.py index af12187c6f..ace6126ebe 100644 --- a/netbox_librenms_plugin/urls.py +++ b/netbox_librenms_plugin/urls.py @@ -1,6 +1,6 @@ from django.urls import include, path -from .models import InterfaceTypeMapping +from .models import DeviceTypeMapping, InterfaceTypeMapping, ModuleBayMapping, ModuleTypeMapping, NormalizationRule from .views import ( AddDeviceToLibreNMSView, AssignVCSerialView, @@ -10,15 +10,27 @@ DeleteNetBoxInterfacesView, DeviceCableTableView, DeviceClusterUpdateView, + DeviceConflictActionView, DeviceInterfaceTableView, DeviceIPAddressTableView, DeviceLibreNMSSyncView, + DeviceModuleTableView, DeviceRackUpdateView, DeviceRoleUpdateView, DeviceStatusListView, + DeviceTypeMappingBulkDeleteView, + DeviceTypeMappingBulkImportView, + DeviceTypeMappingChangeLogView, + DeviceTypeMappingCreateView, + DeviceTypeMappingDeleteView, + DeviceTypeMappingEditView, + DeviceTypeMappingListView, + DeviceTypeMappingView, DeviceValidationDetailsView, DeviceVCDetailsView, DeviceVLANTableView, + InstallBranchView, + InstallModuleView, InterfaceTypeMappingBulkDeleteView, InterfaceTypeMappingBulkImportView, InterfaceTypeMappingChangeLogView, @@ -29,6 +41,31 @@ InterfaceTypeMappingView, LibreNMSImportView, LibreNMSSettingsView, + ModuleBayMappingBulkDeleteView, + ModuleBayMappingBulkImportView, + ModuleBayMappingChangeLogView, + ModuleBayMappingCreateView, + ModuleBayMappingDeleteView, + ModuleBayMappingEditView, + ModuleBayMappingListView, + ModuleBayMappingView, + ModuleTypeMappingBulkDeleteView, + ModuleTypeMappingBulkImportView, + ModuleTypeMappingChangeLogView, + ModuleTypeMappingCreateView, + ModuleTypeMappingDeleteView, + ModuleTypeMappingEditView, + ModuleTypeMappingListView, + ModuleTypeMappingView, + NormalizationRuleBulkDeleteView, + NormalizationRuleBulkImportView, + NormalizationRuleChangeLogView, + NormalizationRuleCreateView, + NormalizationRuleDeleteView, + NormalizationRuleEditView, + NormalizationRuleListView, + NormalizationRuleView, + RemoveServerMappingView, SaveUserPrefView, SingleCableVerifyView, SingleInterfaceVerifyView, @@ -43,6 +80,7 @@ SyncVLANsView, TestLibreNMSConnectionView, UpdateDeviceLocationView, + UpdateDeviceNameView, UpdateDevicePlatformView, UpdateDeviceSerialView, UpdateDeviceTypeView, @@ -69,6 +107,21 @@ DeviceCableTableView.as_view(), name="device_cable_sync", ), + path( + "devices//module-sync/", + DeviceModuleTableView.as_view(), + name="device_module_sync", + ), + path( + "devices//install-module/", + InstallModuleView.as_view(), + name="install_module", + ), + path( + "devices//install-branch/", + InstallBranchView.as_view(), + name="install_branch", + ), path( "devices//ipaddress-sync/", DeviceIPAddressTableView.as_view(), @@ -188,7 +241,12 @@ UpdateDeviceLocationView.as_view(), name="update_device_location", ), - # Update device field URLs (serial, device type, platform) + # Update device field URLs (name, serial, device type, platform) + path( + "devices//update-name/", + UpdateDeviceNameView.as_view(), + name="update_device_name", + ), path( "devices//update-serial/", UpdateDeviceSerialView.as_view(), @@ -214,6 +272,11 @@ AssignVCSerialView.as_view(), name="assign_vc_serial", ), + path( + "devices//remove-server-mapping/", + RemoveServerMappingView.as_view(), + name="remove_server_mapping", + ), path( "device-status/", DeviceStatusListView.as_view(), @@ -259,6 +322,11 @@ DeviceRackUpdateView.as_view(), name="device_rack_update", ), + path( + "device-import/conflict-action//", + DeviceConflictActionView.as_view(), + name="device_conflict_action", + ), path( "save-user-pref/", SaveUserPrefView.as_view(), @@ -323,5 +391,173 @@ InterfaceTypeMappingBulkDeleteView.as_view(), name="interfacetypemapping_bulk_delete", ), + # Device type mapping URLs + path( + "device-type-mappings/", + DeviceTypeMappingListView.as_view(), + name="devicetypemapping_list", + ), + path( + "device-type-mappings//", + DeviceTypeMappingView.as_view(), + name="devicetypemapping_detail", + ), + path( + "device-type-mappings/add/", + DeviceTypeMappingCreateView.as_view(), + name="devicetypemapping_add", + ), + path( + "device-type-mappings/import/", + DeviceTypeMappingBulkImportView.as_view(), + name="devicetypemapping_bulk_import", + ), + path( + "device-type-mappings//delete/", + DeviceTypeMappingDeleteView.as_view(), + name="devicetypemapping_delete", + ), + path( + "device-type-mappings//edit/", + DeviceTypeMappingEditView.as_view(), + name="devicetypemapping_edit", + ), + path( + "device-type-mappings//changelog/", + DeviceTypeMappingChangeLogView.as_view(), + name="devicetypemapping_changelog", + kwargs={"model": DeviceTypeMapping}, + ), + path( + "device-type-mappings/delete/", + DeviceTypeMappingBulkDeleteView.as_view(), + name="devicetypemapping_bulk_delete", + ), + # Module type mapping URLs + path( + "module-type-mappings/", + ModuleTypeMappingListView.as_view(), + name="moduletypemapping_list", + ), + path( + "module-type-mappings//", + ModuleTypeMappingView.as_view(), + name="moduletypemapping_detail", + ), + path( + "module-type-mappings/add/", + ModuleTypeMappingCreateView.as_view(), + name="moduletypemapping_add", + ), + path( + "module-type-mappings/import/", + ModuleTypeMappingBulkImportView.as_view(), + name="moduletypemapping_bulk_import", + ), + path( + "module-type-mappings//delete/", + ModuleTypeMappingDeleteView.as_view(), + name="moduletypemapping_delete", + ), + path( + "module-type-mappings//edit/", + ModuleTypeMappingEditView.as_view(), + name="moduletypemapping_edit", + ), + path( + "module-type-mappings//changelog/", + ModuleTypeMappingChangeLogView.as_view(), + name="moduletypemapping_changelog", + kwargs={"model": ModuleTypeMapping}, + ), + path( + "module-type-mappings/delete/", + ModuleTypeMappingBulkDeleteView.as_view(), + name="moduletypemapping_bulk_delete", + ), + # Module Bay Mapping URLs + path( + "module-bay-mappings/", + ModuleBayMappingListView.as_view(), + name="modulebaymapping_list", + ), + path( + "module-bay-mappings//", + ModuleBayMappingView.as_view(), + name="modulebaymapping_detail", + ), + path( + "module-bay-mappings/add/", + ModuleBayMappingCreateView.as_view(), + name="modulebaymapping_add", + ), + path( + "module-bay-mappings/import/", + ModuleBayMappingBulkImportView.as_view(), + name="modulebaymapping_bulk_import", + ), + path( + "module-bay-mappings//delete/", + ModuleBayMappingDeleteView.as_view(), + name="modulebaymapping_delete", + ), + path( + "module-bay-mappings//edit/", + ModuleBayMappingEditView.as_view(), + name="modulebaymapping_edit", + ), + path( + "module-bay-mappings//changelog/", + ModuleBayMappingChangeLogView.as_view(), + name="modulebaymapping_changelog", + kwargs={"model": ModuleBayMapping}, + ), + path( + "module-bay-mappings/delete/", + ModuleBayMappingBulkDeleteView.as_view(), + name="modulebaymapping_bulk_delete", + ), + # Normalization Rule URLs + path( + "normalization-rules/", + NormalizationRuleListView.as_view(), + name="normalizationrule_list", + ), + path( + "normalization-rules//", + NormalizationRuleView.as_view(), + name="normalizationrule_detail", + ), + path( + "normalization-rules/add/", + NormalizationRuleCreateView.as_view(), + name="normalizationrule_add", + ), + path( + "normalization-rules/import/", + NormalizationRuleBulkImportView.as_view(), + name="normalizationrule_bulk_import", + ), + path( + "normalization-rules//delete/", + NormalizationRuleDeleteView.as_view(), + name="normalizationrule_delete", + ), + path( + "normalization-rules//edit/", + NormalizationRuleEditView.as_view(), + name="normalizationrule_edit", + ), + path( + "normalization-rules//changelog/", + NormalizationRuleChangeLogView.as_view(), + name="normalizationrule_changelog", + kwargs={"model": NormalizationRule}, + ), + path( + "normalization-rules/delete/", + NormalizationRuleBulkDeleteView.as_view(), + name="normalizationrule_bulk_delete", + ), path("api/", include("netbox_librenms_plugin.api.urls")), ] diff --git a/netbox_librenms_plugin/utils.py b/netbox_librenms_plugin/utils.py index 4a5bf113a4..d30ccbba99 100644 --- a/netbox_librenms_plugin/utils.py +++ b/netbox_librenms_plugin/utils.py @@ -1,13 +1,123 @@ +import logging import re from typing import Optional from dcim.models import Device +from django.db.models import Q from django.core.exceptions import ObjectDoesNotExist from django.http import HttpRequest from netbox.config import get_config from netbox.plugins import get_plugin_config from utilities.paginator import get_paginate_count as netbox_get_paginate_count +logger = logging.getLogger(__name__) + + +def get_librenms_device_id(obj, server_key: str = "default"): + """ + Get the LibreNMS device/port ID for a specific server from the JSON custom field. + + Supports both the legacy integer format and the new multi-server JSON format:: + + Legacy: librenms_id = 42 β†’ returns 42 for any server_key + New: librenms_id = {"primary": 42} β†’ returns 42 only for server_key="primary" + + Args: + obj: NetBox object with a ``librenms_id`` custom field. + server_key: LibreNMS server key (from plugin ``servers`` config). + + Returns: + int or None + """ + cf_value = obj.cf.get("librenms_id") + if cf_value is None: + return None + if isinstance(cf_value, int): + return cf_value # backward compat: bare integer from pre-migration + if isinstance(cf_value, dict): + return cf_value.get(server_key) + return None + + +def set_librenms_device_id(obj, device_id, server_key: str = "default"): + """ + Set the LibreNMS device/port ID for a specific server on the JSON custom field. + + Migrates any legacy bare-integer value to the dict format on first write. + + Args: + obj: NetBox object with a ``librenms_id`` custom field. + device_id: LibreNMS device ID (integer). + server_key: LibreNMS server key (from plugin ``servers`` config). + """ + cf_value = obj.custom_field_data.get("librenms_id") or {} + if isinstance(cf_value, int): + cf_value = {"default": cf_value} # migrate legacy value on first write + elif not isinstance(cf_value, dict): + logger.warning( + "librenms_id custom field has unexpected type %s on %r; resetting to empty dict.", + type(cf_value).__name__, + obj, + ) + cf_value = {} + cf_value[server_key] = device_id + obj.custom_field_data["librenms_id"] = cf_value + + +def find_by_librenms_id(model, librenms_id, server_key: str = "default"): + """ + Return the first object of *model* whose ``librenms_id`` JSON field contains + *librenms_id* under *server_key*. + + Also matches legacy records that stored ``librenms_id`` as a bare integer + directly in ``custom_field_data``. + + Args: + model: A Django model class (Device, VirtualMachine, Interface, …). + librenms_id: The LibreNMS device/port ID to look up. + server_key: LibreNMS server key (from plugin ``servers`` config). + + Returns: + Model instance or None + """ + return model.objects.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}) + | Q(custom_field_data__librenms_id=librenms_id) + ).first() + + +def migrate_legacy_librenms_id(obj, server_key: str = "default") -> bool: + """ + Migrate a legacy bare-integer ``librenms_id`` custom field to the JSON dict format, + scoped to *server_key*. + + Only performs the migration when the current value is a bare integer, i.e. a record + created before the multi-server JSON refactor. The integer is assumed to belong to + the server identified by *server_key* (the caller must verify this, e.g. by confirming + that the LibreNMS device ID and serial number both match). + + Does **not** call ``obj.save()`` β€” the caller is responsible for persisting the change. + + Args: + obj: NetBox object with a ``librenms_id`` custom field. + server_key: LibreNMS server key the legacy integer should be scoped to. + + Returns: + True if the value was migrated, False if it was already in the correct format. + """ + cf_value = obj.custom_field_data.get("librenms_id") + if not isinstance(cf_value, int): + return False + obj.custom_field_data["librenms_id"] = {server_key: cf_value} + logger.info( + "Migrated legacy librenms_id %d β†’ {%r: %d} on %r", + cf_value, + server_key, + cf_value, + obj, + ) + return True + def convert_speed_to_kbps(speed_bps: int) -> int: """ @@ -193,7 +303,8 @@ def match_librenms_hardware_to_device_type(hardware_name: str) -> dict: """ Match LibreNMS hardware string to a NetBox DeviceType. - Only performs exact matching on part_number and model fields (case-insensitive). + Checks DeviceTypeMapping table first, then falls back to exact matching + on part_number and model fields (case-insensitive). Args: hardware_name (str): Hardware string from LibreNMS API (e.g., 'C9200L-48P-4X') @@ -202,13 +313,29 @@ def match_librenms_hardware_to_device_type(hardware_name: str) -> dict: dict: Dictionary containing: - matched (bool): Whether a match was found - device_type (DeviceType|None): The matched DeviceType object - - match_type (str|None): Always 'exact' if found, None otherwise + - match_type (str|None): 'mapping' if via DeviceTypeMapping, 'exact' if via + part_number/model, None otherwise """ from dcim.models import DeviceType + from netbox_librenms_plugin.models import DeviceTypeMapping + if not hardware_name or hardware_name == "-": return {"matched": False, "device_type": None, "match_type": None} + # Check DeviceTypeMapping table first + try: + mapping = DeviceTypeMapping.objects.get(librenms_hardware__iexact=hardware_name) + return { + "matched": True, + "device_type": mapping.netbox_device_type, + "match_type": "mapping", + } + except DeviceTypeMapping.DoesNotExist: + pass + except DeviceTypeMapping.MultipleObjectsReturned: + pass + # Try part number exact match try: device_type = DeviceType.objects.get(part_number__iexact=hardware_name) @@ -447,3 +574,123 @@ def check_vlan_group_matches( netbox_gid = netbox_tagged_group_ids.get(vid) return netbox_gid == selected_group_id return True + + +# Minimum NetBox version that supports {module_path} token in module templates + + +def supports_module_path(): + """Check if the running NetBox supports the {module_path} template token. + + Detects by checking for MODULE_PATH_TOKEN in dcim.constants rather than + comparing version strings β€” works with patched/pre-release builds too. + """ + try: + from dcim.constants import MODULE_PATH_TOKEN # noqa: F401 + + return True + except ImportError: + return False + + +def module_type_uses_module_path(module_type): + """Check if a ModuleType has any interface templates using {module_path}.""" + return any("{module_path}" in t.name for t in module_type.interfacetemplates.all()) + + +def module_type_uses_module_token(module_type) -> bool: + """Check if a ModuleType has interface templates using the {module} token.""" + try: + from dcim.constants import MODULE_TOKEN + except ImportError: + return False + return any(MODULE_TOKEN in t.name for t in module_type.interfacetemplates.all()) + + +def module_type_is_end_module(module_type) -> bool: + """Return True if this module type defines no module bay templates (i.e., it is a leaf/end module).""" + return not module_type.modulebaytemplates.exists() + + +def has_nested_name_conflict(module_type, module_bay): + """Check if installing this module type in a nested bay would cause a name conflict. + + Returns True when ALL of the following are true: + - The module type has interface templates using only ``{module}`` (not ``{module_path}``) + - The bay is nested (its parent is owned by an installed module) + - There is at least one sibling bay under the same parent + + In this situation NetBox's ``resolve_name()`` replaces ``{module}`` with the + root ancestor's bay position, producing the same interface name for every + sibling at this nesting level. + """ + from dcim.constants import MODULE_TOKEN + + if not module_bay or not module_bay.module_id: + return False # Top-level bay β€” no conflict + + templates = list(module_type.interfacetemplates.all()) + if not templates: + return False # No interface templates + + uses_module_token = any(MODULE_TOKEN in t.name for t in templates) + if not uses_module_token: + return False # Template doesn't use {module} + + # Count how many unique interface names this template would produce across siblings + # If all siblings resolve to the same name, there's a conflict + from dcim.models import ModuleBay as ModuleBayModel + + sibling_count = ModuleBayModel.objects.filter( + device=module_bay.device, + module_id=module_bay.module_id, + ).count() + + return sibling_count > 1 + + +def apply_normalization_rules(value: str, scope: str, manufacturer=None) -> str: + """Apply NormalizationRule chain to transform a string before matching. + + Rules for the given scope are applied in priority order. Each rule's + regex substitution transforms the output of the previous rule, forming + a pipeline. If no rules match, the original value is returned unchanged. + + When *manufacturer* is given, manufacturer-scoped rules run first, + followed by unscoped (manufacturer=NULL) rules. When *manufacturer* + is ``None``, all rules for the scope run in priority order. + + Args: + value: The raw string to normalize (e.g. '3HE16474AARA01'). + scope: One of NormalizationRule.SCOPE_* constants. + manufacturer: Optional Manufacturer instance to scope rules. + + Returns: + The normalized string after all matching rules have been applied. + """ + from netbox_librenms_plugin.models import NormalizationRule + + if not value: + return value + + if manufacturer: + # Manufacturer-specific rules first, then unscoped rules + for mfg_filter in [{"manufacturer": manufacturer}, {"manufacturer__isnull": True}]: + rules = NormalizationRule.objects.filter(scope=scope, **mfg_filter).order_by("priority", "pk") + for rule in rules: + try: + value = re.sub(rule.match_pattern, rule.replacement, value) + except re.error: + logger.error( + "Invalid regex in NormalizationRule pk=%s pattern=%r β€” skipping", rule.pk, rule.match_pattern + ) + else: + rules = NormalizationRule.objects.filter(scope=scope).order_by("priority", "pk") + for rule in rules: + try: + value = re.sub(rule.match_pattern, rule.replacement, value) + except re.error: + logger.error( + "Invalid regex in NormalizationRule pk=%s pattern=%r β€” skipping", rule.pk, rule.match_pattern + ) + return value diff --git a/netbox_librenms_plugin/views/__init__.py b/netbox_librenms_plugin/views/__init__.py index d2b3bbdd43..bb27fde9f3 100644 --- a/netbox_librenms_plugin/views/__init__.py +++ b/netbox_librenms_plugin/views/__init__.py @@ -1,16 +1,22 @@ """ Module for initializing views for the NetBox LibreNMS plugin. + +All imports below are intentional re-exports consumed by urls.py and +other modules. The F401 suppressions prevent linters from flagging +them as unused within this file. """ -from .base.cables_view import BaseCableTableView, SingleCableVerifyView -from .base.interfaces_view import BaseInterfaceTableView -from .base.ip_addresses_view import BaseIPAddressTableView, SingleIPAddressVerifyView -from .base.librenms_sync_view import BaseLibreNMSSyncView -from .base.vlan_table_view import BaseVLANTableView -from .imports import ( +from .base.cables_view import BaseCableTableView, SingleCableVerifyView # noqa: F401 +from .base.interfaces_view import BaseInterfaceTableView # noqa: F401 +from .base.ip_addresses_view import BaseIPAddressTableView, SingleIPAddressVerifyView # noqa: F401 +from .base.librenms_sync_view import BaseLibreNMSSyncView # noqa: F401 +from .base.modules_view import InstallBranchView, InstallModuleView # noqa: F401 +from .base.vlan_table_view import BaseVLANTableView # noqa: F401 +from .imports import ( # noqa: F401 BulkImportConfirmView, BulkImportDevicesView, DeviceClusterUpdateView, + DeviceConflictActionView, DeviceRackUpdateView, DeviceRoleUpdateView, DeviceValidationDetailsView, @@ -18,7 +24,15 @@ LibreNMSImportView, SaveUserPrefView, ) -from .mapping_views import ( +from .mapping_views import ( # noqa: F401 + DeviceTypeMappingBulkDeleteView, + DeviceTypeMappingBulkImportView, + DeviceTypeMappingChangeLogView, + DeviceTypeMappingCreateView, + DeviceTypeMappingDeleteView, + DeviceTypeMappingEditView, + DeviceTypeMappingListView, + DeviceTypeMappingView, InterfaceTypeMappingBulkDeleteView, InterfaceTypeMappingBulkImportView, InterfaceTypeMappingChangeLogView, @@ -27,12 +41,37 @@ InterfaceTypeMappingEditView, InterfaceTypeMappingListView, InterfaceTypeMappingView, + ModuleBayMappingBulkDeleteView, + ModuleBayMappingBulkImportView, + ModuleBayMappingChangeLogView, + ModuleBayMappingCreateView, + ModuleBayMappingDeleteView, + ModuleBayMappingEditView, + ModuleBayMappingListView, + ModuleBayMappingView, + ModuleTypeMappingBulkDeleteView, + ModuleTypeMappingBulkImportView, + ModuleTypeMappingChangeLogView, + ModuleTypeMappingCreateView, + ModuleTypeMappingDeleteView, + ModuleTypeMappingEditView, + ModuleTypeMappingListView, + ModuleTypeMappingView, + NormalizationRuleBulkDeleteView, + NormalizationRuleBulkImportView, + NormalizationRuleChangeLogView, + NormalizationRuleCreateView, + NormalizationRuleDeleteView, + NormalizationRuleEditView, + NormalizationRuleListView, + NormalizationRuleView, ) -from .object_sync import ( +from .object_sync import ( # noqa: F401 DeviceCableTableView, DeviceInterfaceTableView, DeviceIPAddressTableView, DeviceLibreNMSSyncView, + DeviceModuleTableView, DeviceVLANTableView, SaveVlanGroupOverridesView, SingleInterfaceVerifyView, @@ -42,18 +81,20 @@ VMIPAddressTableView, VMLibreNMSSyncView, ) -from .settings_views import LibreNMSSettingsView, TestLibreNMSConnectionView -from .status_check import DeviceStatusListView, VMStatusListView -from .sync.cables import SyncCablesView -from .sync.device_fields import ( +from .settings_views import LibreNMSSettingsView, TestLibreNMSConnectionView # noqa: F401 +from .status_check import DeviceStatusListView, VMStatusListView # noqa: F401 +from .sync.cables import SyncCablesView # noqa: F401 +from .sync.device_fields import ( # noqa: F401 AssignVCSerialView, CreateAndAssignPlatformView, + RemoveServerMappingView, + UpdateDeviceNameView, UpdateDevicePlatformView, UpdateDeviceSerialView, UpdateDeviceTypeView, ) -from .sync.devices import AddDeviceToLibreNMSView, UpdateDeviceLocationView -from .sync.interfaces import DeleteNetBoxInterfacesView, SyncInterfacesView -from .sync.ip_addresses import SyncIPAddressesView -from .sync.locations import SyncSiteLocationView -from .sync.vlans import SyncVLANsView +from .sync.devices import AddDeviceToLibreNMSView, UpdateDeviceLocationView # noqa: F401 +from .sync.interfaces import DeleteNetBoxInterfacesView, SyncInterfacesView # noqa: F401 +from .sync.ip_addresses import SyncIPAddressesView # noqa: F401 +from .sync.locations import SyncSiteLocationView # noqa: F401 +from .sync.vlans import SyncVLANsView # noqa: F401 diff --git a/netbox_librenms_plugin/views/base/cables_view.py b/netbox_librenms_plugin/views/base/cables_view.py index c390cdd539..46fd0f21b2 100644 --- a/netbox_librenms_plugin/views/base/cables_view.py +++ b/netbox_librenms_plugin/views/base/cables_view.py @@ -4,6 +4,7 @@ from django.contrib import messages from django.core.cache import cache from django.core.exceptions import MultipleObjectsReturned +from django.db.models import Q from django.http import JsonResponse from django.shortcuts import get_object_or_404, render from django.urls import reverse @@ -24,7 +25,6 @@ class BaseCableTableView(LibreNMSPermissionMixin, LibreNMSAPIMixin, CacheMixin, model = None # To be defined in subclasses partial_template_name = "netbox_librenms_plugin/_cable_sync_content.html" - interface_name_field = get_interface_name_field() def get_object(self, pk): """Retrieve the object (Device or VirtualMachine).""" @@ -53,11 +53,17 @@ def get_links_data(self, obj): if not success or "error" in data: return None + interface_name_field = get_interface_name_field(getattr(self, "request", None)) ports_data = self.get_ports_data(obj) local_ports_map = {} for port in ports_data.get("ports", []): - port_id = str(port["port_id"]) - port_name = port[self.interface_name_field] + raw_port_id = port.get("port_id") + if raw_port_id is None: + continue + port_id = str(raw_port_id) + port_name = port.get(interface_name_field) + if port_name is None: + continue local_ports_map[port_id] = port_name links = data.get("links", []) @@ -78,10 +84,14 @@ def get_links_data(self, obj): def get_device_by_id_or_name(self, remote_device_id, hostname): """Try to find device in NetBox first by librenms_id custom field, then by name""" + server_key = self.librenms_api.server_key # First try matching by LibreNMS ID if remote_device_id: try: - device = Device.objects.get(custom_field_data__librenms_id=remote_device_id) + device = Device.objects.get( + Q(**{f"custom_field_data__librenms_id__{server_key}": remote_device_id}) + | Q(custom_field_data__librenms_id=remote_device_id) + ) return device, True, None except Device.DoesNotExist: pass @@ -116,13 +126,17 @@ def enrich_local_port(self, link, obj): if local_port := link.get("local_port"): interface = None local_port_id = link.get("local_port_id") + server_key = self.librenms_api.server_key if hasattr(obj, "virtual_chassis") and obj.virtual_chassis: chassis_member = get_virtual_chassis_member(obj, local_port) # First try to find interface by librenms_id if local_port_id: - interface = chassis_member.interfaces.filter(custom_field_data__librenms_id=local_port_id).first() + interface = chassis_member.interfaces.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": local_port_id}) + | Q(custom_field_data__librenms_id=local_port_id) + ).first() # Only if librenms_id match fails, try matching by name if not interface: @@ -130,7 +144,10 @@ def enrich_local_port(self, link, obj): else: # First try to find interface by librenms_id if local_port_id: - interface = obj.interfaces.filter(custom_field_data__librenms_id=local_port_id).first() + interface = obj.interfaces.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": local_port_id}) + | Q(custom_field_data__librenms_id=local_port_id) + ).first() # Only if librenms_id match fails, try matching by name if not interface: @@ -145,6 +162,7 @@ def enrich_remote_port(self, link, device): if remote_port := link.get("remote_port"): netbox_remote_interface = None librenms_remote_port_id = link.get("remote_port_id") + server_key = self.librenms_api.server_key # Handle virtual chassis case if hasattr(device, "virtual_chassis") and device.virtual_chassis: @@ -154,7 +172,8 @@ def enrich_remote_port(self, link, device): # First try to find interface by librenms_id if librenms_remote_port_id: netbox_remote_interface = chassis_member.interfaces.filter( - custom_field_data__librenms_id=librenms_remote_port_id + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id}) + | Q(custom_field_data__librenms_id=librenms_remote_port_id) ).first() # If not found by librenms_id, fall back to name matching on the correct chassis member @@ -165,7 +184,8 @@ def enrich_remote_port(self, link, device): # First try to find interface by librenms_id if librenms_remote_port_id: netbox_remote_interface = device.interfaces.filter( - custom_field_data__librenms_id=librenms_remote_port_id + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id}) + | Q(custom_field_data__librenms_id=librenms_remote_port_id) ).first() # If not found by librenms_id, fall back to name matching @@ -369,8 +389,10 @@ def post(self, request): # First try to find interface by librenms_id interface = None if local_port_id := link_data.get("local_port_id"): + _sk = self.librenms_api.server_key interface = selected_device.interfaces.filter( - custom_field_data__librenms_id=local_port_id + Q(**{f"custom_field_data__librenms_id__{_sk}": local_port_id}) + | Q(custom_field_data__librenms_id=local_port_id) ).first() # If not found by librenms_id, try matching by name diff --git a/netbox_librenms_plugin/views/base/ip_addresses_view.py b/netbox_librenms_plugin/views/base/ip_addresses_view.py index 22f4b49742..1ca3f6e290 100644 --- a/netbox_librenms_plugin/views/base/ip_addresses_view.py +++ b/netbox_librenms_plugin/views/base/ip_addresses_view.py @@ -11,7 +11,7 @@ from virtualization.models import VirtualMachine from netbox_librenms_plugin.tables.ipaddresses import IPAddressTable -from netbox_librenms_plugin.utils import get_interface_name_field +from netbox_librenms_plugin.utils import get_interface_name_field, get_librenms_device_id from netbox_librenms_plugin.views.mixins import CacheMixin, LibreNMSAPIMixin, LibreNMSPermissionMixin @@ -104,10 +104,11 @@ def _prefetch_netbox_data(self, obj): all_interfaces = list(obj.interfaces.all()) # Create maps for efficient lookups + server_key = self.librenms_api.server_key interfaces_by_librenms_id = { - interface.custom_field_data.get("librenms_id"): interface + get_librenms_device_id(interface, server_key): interface for interface in all_interfaces - if interface.custom_field_data.get("librenms_id") + if get_librenms_device_id(interface, server_key) } interfaces_by_name = {interface.name: interface for interface in all_interfaces} diff --git a/netbox_librenms_plugin/views/base/librenms_sync_view.py b/netbox_librenms_plugin/views/base/librenms_sync_view.py index 1346c3cb13..77566a2dd0 100644 --- a/netbox_librenms_plugin/views/base/librenms_sync_view.py +++ b/netbox_librenms_plugin/views/base/librenms_sync_view.py @@ -1,5 +1,6 @@ import re +from django.conf import settings as django_settings from django.shortcuts import get_object_or_404, render from netbox.views import generic @@ -86,6 +87,7 @@ def get_context_data(self, request, obj): cable_context = self.get_cable_context(request, obj) ip_context = self.get_ip_context(request, obj) vlan_context = self.get_vlan_context(request, obj) + module_context = self.get_module_context(request, obj) interface_name_field = get_interface_name_field(request) @@ -103,6 +105,7 @@ def get_context_data(self, request, obj): "cable_sync": cable_context, "ip_sync": ip_context, "vlan_sync": vlan_context, + "module_sync": module_context, "v1v2form": AddToLIbreSNMPV1V2(prefix="v1v2"), "v3form": AddToLIbreSNMPV3(prefix="v3"), "librenms_device_id": self.librenms_id, @@ -114,6 +117,7 @@ def get_context_data(self, request, obj): "platform_info": platform_info, "vc_inventory_serials": librenms_info["librenms_device_details"].get("vc_inventory_serials", []), "manufacturers": manufacturers, + "all_server_mappings": self._build_all_server_mappings(obj, self.librenms_api.server_key), } ) @@ -230,6 +234,7 @@ def get_librenms_device_info(self, obj): if netbox_identities & librenms_identities: mismatched_device = False else: + # Device is still found (we have librenms_id), just mismatched mismatched_device = True librenms_device_details["netbox_dns_name"] = netbox_dns_name or "-" @@ -268,6 +273,61 @@ def get_vlan_context(self, request, obj): """ return None + def get_module_context(self, request, obj): + """ + Get the context data for module sync. + Subclasses should override this method if applicable. + """ + return None + + @staticmethod + def _build_all_server_mappings(obj, active_server_key): + """Build a list of all LibreNMS server mappings for the given device. + + Each entry describes one server<->ID mapping stored in the ``librenms_id`` + custom field: + + * ``server_key`` – the key as stored in the CF dict (or ``"legacy"``). + * ``display_name`` – human-readable name from PLUGINS_CONFIG, or the key. + * ``librenms_url`` – base URL of that server (``None`` when not configured). + * ``device_id`` – the integer device ID on that server. + * ``device_url`` – direct URL to the device page on that server (or ``None``). + * ``is_configured``– True when the server key exists in current plugin config. + * ``is_active`` – True when this is the currently active server. + + Returns ``None`` for legacy bare-int format (no per-server info to show) + and ``None`` when the CF is absent/invalid. + """ + cf_value = obj.custom_field_data.get("librenms_id") + if not isinstance(cf_value, dict) or not cf_value: + return None + + plugins_cfg = django_settings.PLUGINS_CONFIG.get("netbox_librenms_plugin", {}) + servers_config = plugins_cfg.get("servers", {}) + + result = [] + for sk, did in cf_value.items(): + srv_cfg = servers_config.get(sk) + is_configured = srv_cfg is not None + librenms_url = srv_cfg.get("librenms_url") if srv_cfg else None + display_name = (srv_cfg.get("display_name") or sk) if srv_cfg else sk + device_url = f"{librenms_url}/device/device={did}/" if librenms_url else None + result.append( + { + "server_key": sk, + "display_name": display_name, + "librenms_url": librenms_url, + "device_id": did, + "device_url": device_url, + "is_configured": is_configured, + "is_active": sk == active_server_key, + } + ) + + # Sort: active first, then configured, then orphaned + result.sort(key=lambda e: (0 if e["is_active"] else (1 if e["is_configured"] else 2))) + return result or None + @staticmethod def _strip_vc_pattern(name): """Strip the VC member naming suffix from a device name. diff --git a/netbox_librenms_plugin/views/base/modules_view.py b/netbox_librenms_plugin/views/base/modules_view.py new file mode 100644 index 0000000000..68a94b2fca --- /dev/null +++ b/netbox_librenms_plugin/views/base/modules_view.py @@ -0,0 +1,1064 @@ +from django.contrib import messages +from django.core.cache import cache +from django.db import transaction +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse +from django.utils import timezone +from django.views import View + +from netbox_librenms_plugin.views.mixins import ( + CacheMixin, + LibreNMSAPIMixin, + LibreNMSPermissionMixin, + NetBoxObjectPermissionMixin, +) + + +# entPhysicalClass values relevant for module sync +# Includes vendor-specific classes (Nokia TIMETRA-CHASSIS-MIB uses ioModule, cpmModule, etc.) +INVENTORY_CLASSES = { + "module", + "powerSupply", + "fan", + "port", + "container", + "ioModule", + "cpmModule", + "mdaModule", + "fabricModule", + "xioModule", +} + +# Model name values that indicate a generic/empty container (not real hardware) +_GENERIC_CONTAINER_MODELS = {"", "BUILTIN", "Default", "N/A"} + + +class BaseModuleTableView(LibreNMSPermissionMixin, LibreNMSAPIMixin, CacheMixin, View): + """ + Base view for synchronizing module/inventory data from LibreNMS. + Fetches inventory, matches against NetBox module bays and module types, + and renders a comparison table. + """ + + model = None + partial_template_name = "netbox_librenms_plugin/_module_sync_content.html" + + def get_object(self, pk): + """Retrieve the object (Device).""" + return get_object_or_404(self.model, pk=pk) + + def get_table(self, data, obj): + """Returns the table class. Subclasses should override.""" + raise NotImplementedError("Subclasses must implement get_table()") + + def post(self, request, pk): + """Fetch inventory from LibreNMS, cache it, and render the module sync table.""" + obj = self.get_object(pk) + + self.librenms_id = self.librenms_api.get_librenms_id(obj) + if not self.librenms_id: + messages.error(request, "Device not found in LibreNMS.") + return render( + request, + self.partial_template_name, + {"module_sync": {"object": obj, "table": None, "cache_expiry": None}}, + ) + + success, inventory_data = self.librenms_api.get_device_inventory(self.librenms_id) + + if not success: + messages.error(request, f"Failed to fetch inventory from LibreNMS: {inventory_data}") + return render( + request, + self.partial_template_name, + {"module_sync": {"object": obj, "table": None, "cache_expiry": None}}, + ) + + # Fetch transceiver data and merge with inventory + inventory_data = self._merge_transceiver_data(inventory_data) + + # Cache the merged inventory data + cache.set( + self.get_cache_key(obj, "inventory"), + inventory_data, + timeout=self.librenms_api.cache_timeout, + ) + + context = self._build_context(request, obj, inventory_data) + messages.success(request, "Inventory data refreshed successfully.") + return render(request, self.partial_template_name, {"module_sync": context}) + + def get_context_data(self, request, obj): + """Get context from cache (used by the main sync view on initial page load).""" + cached_data = cache.get(self.get_cache_key(obj, "inventory")) + if not cached_data: + return {"table": None, "object": obj, "cache_expiry": None} + return self._build_context(request, obj, cached_data) + + def _build_context(self, request, obj, inventory_data): + """Build context with matched inventory items and table.""" + # Build a lookup of all inventory items by index for parent resolution + index_map = {item["entPhysicalIndex"]: item for item in inventory_data} + + # Store manufacturer for normalization rules in _build_row + self._device_manufacturer = getattr(getattr(obj, "device_type", None), "manufacturer", None) + + # Get NetBox module bays and modules for this device + device_bays, module_scoped_bays = self._get_module_bays(obj) + module_types = self._get_module_types() + + # Collect top-level items and their sub-components + # Include synthetic transceiver items (from vendors without ENTITY-MIB SFP data) + # Exclude items that have any ancestor with an INVENTORY_CLASSES class + # (they appear as sub-components under that ancestor) + top_items = [] + for item in inventory_data: + if item.get("_from_transceiver_api"): + top_items.append(item) + continue + phys_class = item.get("entPhysicalClass") + if phys_class not in INVENTORY_CLASSES: + continue + # Skip items with generic model names (not real hardware). + # Containers with empty model are physical slot representations. + model = (item.get("entPhysicalModelName") or "").strip() + if phys_class == "container" and model in _GENERIC_CONTAINER_MODELS: + continue + if model and model in _GENERIC_CONTAINER_MODELS: + continue + # Walk up ancestor chain; skip if any ancestor is an inventory-class item. + # Containers with empty model are physical slot/bay representations, not + # real modules β€” skip them so children can be top-level items. + is_descendant = False + current_idx = item.get("entPhysicalContainedIn", 0) + for _ in range(10): + if not current_idx or current_idx not in index_map: + break + ancestor = index_map[current_idx] + anc_class = ancestor.get("entPhysicalClass") + if anc_class in INVENTORY_CLASSES: + anc_model = (ancestor.get("entPhysicalModelName") or "").strip() + # Empty-model containers are just physical slot representations + if anc_class == "container" and not anc_model: + current_idx = ancestor.get("entPhysicalContainedIn", 0) + continue + is_descendant = True + break + current_idx = ancestor.get("entPhysicalContainedIn", 0) + if is_descendant: + continue + top_items.append(item) + + table_data = [] + from netbox_librenms_plugin.utils import apply_normalization_rules + + # Build combined bay lookup so synthetic transceiver entries (which may + # live inside installed modules) can find their module-scoped bays. + all_bays = dict(device_bays) + for scope_bays in module_scoped_bays.values(): + all_bays.update(scope_bays) + + for item in top_items: + # Transceiver API entries may live inside installed modules, so they + # need the full bay map. ENTITY-MIB top-level items must only match + # device-level bays to avoid name collisions with module-scoped bays + # that share the same name as a device bay. + item_bays = all_bays if item.get("_from_transceiver_api") else device_bays + row = self._build_row(item, index_map, item_bays, module_types, depth=0) + parent_idx = len(table_data) + table_data.append(row) + + # Determine which bays sub-components should match against: + # If parent matched a bay with an installed module, use that module's child bays. + # If parent matched a bay but it's NOT installed, children can't be installed + # individually (parent must be installed first to create child bays). + parent_module_id = None + parent_bay_matched_but_uninstalled = False + if row.get("module_bay_id"): + matched_bay = item_bays.get(row["module_bay"]) + if matched_bay and hasattr(matched_bay, "installed_module") and matched_bay.installed_module: + parent_module_id = matched_bay.installed_module.pk + else: + # Parent matched a bay but it's not installed yet + parent_bay_matched_but_uninstalled = True + + if parent_bay_matched_but_uninstalled: + # Empty dict: children can't match any bay individually + child_bays = {} + elif parent_module_id: + child_bays = module_scoped_bays.get(parent_module_id, {}) + else: + child_bays = device_bays + + # Find sub-components with a model name (transceivers, converters, etc.) + # Track bay scope per depth level so nested modules use correct bays + bays_by_depth = {0: child_bays} + sub_items = self._get_sub_components(item["entPhysicalIndex"], inventory_data) + for depth, sub_item in sub_items: + scope_bays = bays_by_depth.get(depth, child_bays) + sub_row = self._build_row(sub_item, index_map, scope_bays, module_types, depth=depth) + table_data.append(sub_row) + + # If this sub-item matched an installed module, deeper items use its bays + if sub_row.get("module_bay_id"): + matched_sub_bay = scope_bays.get(sub_row["module_bay"]) + if ( + matched_sub_bay + and hasattr(matched_sub_bay, "installed_module") + and matched_sub_bay.installed_module + ): + sub_module_id = matched_sub_bay.installed_module.pk + bays_by_depth[depth + 1] = module_scoped_bays.get(sub_module_id, {}) + + # Mark parent if any child is installable + if sub_row.get("can_install"): + table_data[parent_idx]["has_installable_children"] = True + + # When parent is installable but children can't match bays yet + # (parent module not installed), enable "Install Branch" if any child + # has a matching module type (branch install handles bay creation). + if ( + parent_bay_matched_but_uninstalled + and row.get("can_install") + and not table_data[parent_idx].get("has_installable_children") + ): + for _depth, sub_item in sub_items: + sub_model = (sub_item.get("entPhysicalModelName") or "").strip() + if not sub_model: + continue + matched = module_types.get(sub_model) + if not matched: + normalized = apply_normalization_rules( + sub_model, + "module_type", + manufacturer=getattr(self, "_device_manufacturer", None), + ) + matched = module_types.get(normalized) + if matched: + table_data[parent_idx]["has_installable_children"] = True + break + + # Sort top-level groups by status, keeping children after their parent + table_data = self._sort_with_hierarchy(table_data) + + table = self.get_table(table_data, obj) + table.configure(request) + + cache_ttl = getattr(cache, "ttl", lambda k: None)(self.get_cache_key(obj, "inventory")) + cache_expiry = timezone.now() + timezone.timedelta(seconds=cache_ttl) if cache_ttl is not None else None + + return { + "table": table, + "object": obj, + "cache_expiry": cache_expiry, + } + + def _merge_transceiver_data(self, inventory_data): + """Merge transceiver API data with entity inventory. + + For vendors like Nokia that don't expose SFPs in ENTITY-MIB, + the transceiver API provides SFP model, serial, and type info. + + Strategy: + - For transceivers matching existing inventory items by entity_physical_index: + supplement entPhysicalModelName if empty + - For transceivers NOT in inventory: create synthetic inventory items + so they appear in the modules table + """ + success, transceivers = self.librenms_api.get_device_transceivers(self.librenms_id) + if not success or not transceivers: + return inventory_data + + # Build lookup of existing inventory items by index and serial + inv_by_index = {item["entPhysicalIndex"]: item for item in inventory_data} + inv_serials = { + (item.get("entPhysicalSerialNum") or "").strip() + for item in inventory_data + if (item.get("entPhysicalSerialNum") or "").strip() + } + + # Build port_id β†’ ifName lookup for better synthetic item naming + port_name_map = self._build_port_name_map(transceivers) + + # Types that are containers, not real transceiver modules + SKIP_TYPES = {"Port Container", "Port", ""} + + for txr in transceivers: + ent_idx = txr.get("entity_physical_index") + if not ent_idx: + continue + + model = (txr.get("model") or "").strip() + serial = (txr.get("serial") or "").strip() + txr_type = (txr.get("type") or "").strip() + + # Skip containers and entries with no useful data + if txr_type in SKIP_TYPES and not model and not serial: + continue + + # Use transceiver type as model fallback (e.g., "CFP2/QSFP28") + display_model = model or (txr_type if txr_type not in SKIP_TYPES else "") + + if ent_idx in inv_by_index: + # Supplement existing inventory item if model is missing + existing = inv_by_index[ent_idx] + if not (existing.get("entPhysicalModelName") or "").strip() and display_model: + existing["entPhysicalModelName"] = display_model + if not (existing.get("entPhysicalSerialNum") or "").strip() and serial: + existing["entPhysicalSerialNum"] = serial + else: + # Skip if serial already exists in ENTITY-MIB data (avoid duplicates) + if serial and serial in inv_serials: + continue + # Create synthetic inventory item for SFPs not in entity inventory + port_id = txr.get("port_id", 0) + ifname = port_name_map.get(port_id) + if ifname: + name = ifname + elif port_id: + name = f"Transceiver (port {port_id})" + else: + name = f"Transceiver {ent_idx}" + + synthetic = { + "entPhysicalIndex": ent_idx, + "entPhysicalName": name, + "entPhysicalClass": "port", + "entPhysicalModelName": display_model, + "entPhysicalSerialNum": serial, + "entPhysicalDescr": txr_type, + "entPhysicalContainedIn": 0, + "_from_transceiver_api": True, + } + inventory_data.append(synthetic) + + return inventory_data + + def _build_port_name_map(self, transceivers): + """Build port_id β†’ ifName mapping for transceiver ports. + + Fetches port data from LibreNMS to resolve port IDs to interface names, + enabling better bay matching for synthetic transceiver items (e.g., + Nokia 1/1/c1 instead of opaque port IDs). + """ + port_ids = {txr.get("port_id") for txr in transceivers if txr.get("port_id")} + if not port_ids: + return {} + + success, ports_data = self.librenms_api.get_ports(self.librenms_id) + if not success or not isinstance(ports_data, dict): + return {} + + return { + p["port_id"]: p["ifName"] + for p in ports_data.get("ports", []) + if p.get("port_id") in port_ids and p.get("ifName") + } + + def _get_sub_components(self, parent_idx, inventory_data): + """Find descendant items with a model name (real hardware, not empty containers). + + Returns list of (depth, item) tuples. + """ + results = [] + self._collect_descendants(parent_idx, inventory_data, depth=1, results=results) + return results + + def _collect_descendants(self, parent_idx, inventory_data, depth, results): + """Recursively collect descendant items that have a model name.""" + children = [i for i in inventory_data if i.get("entPhysicalContainedIn") == parent_idx] + for child in children: + model = (child.get("entPhysicalModelName") or "").strip() + if model and model not in _GENERIC_CONTAINER_MODELS: + results.append((depth, child)) + # Continue looking for deeper components (e.g., SFPs inside converters) + self._collect_descendants(child["entPhysicalIndex"], inventory_data, depth + 1, results) + else: + # Skip generic/empty items, but check their children + self._collect_descendants(child["entPhysicalIndex"], inventory_data, depth, results) + + def _sort_with_hierarchy(self, table_data): + """Sort table keeping children grouped under their parent.""" + status_order = {"Installed": 0, "Serial Mismatch": 1, "Matched": 2, "No Type": 3, "No Bay": 4, "Unmatched": 5} + + # Group into top-level items with their children + groups = [] + current_group = None + for row in table_data: + if row.get("depth", 0) == 0: + current_group = {"parent": row, "children": []} + groups.append(current_group) + elif current_group is not None: + current_group["children"].append(row) + + # Sort groups by parent status + groups.sort(key=lambda g: status_order.get(g["parent"]["status"], 99)) + + # Flatten back + result = [] + for group in groups: + result.append(group["parent"]) + result.extend(group["children"]) + return result + + def _get_module_bays(self, obj): + """Get module bays for the device, organized by scope. + + Returns: + tuple: (device_bays, module_bays) where: + - device_bays: {name: bay} for device-level bays (module=None) + - module_bays: {module_id: {name: bay}} for bays created by installed modules + """ + from dcim.models import ModuleBay + + bays = ModuleBay.objects.filter(device=obj).select_related("installed_module__module_type") + device_bays = {} + module_scoped_bays = {} + for bay in bays: + if bay.module_id: + module_scoped_bays.setdefault(bay.module_id, {})[bay.name] = bay + else: + device_bays[bay.name] = bay + return device_bays, module_scoped_bays + + def _get_module_types(self): + """Get all module types, indexed by model (part_number), with ModuleTypeMapping checked first.""" + from dcim.models import ModuleType + + from netbox_librenms_plugin.models import ModuleTypeMapping + + # Build base lookup from NetBox module types + types = ModuleType.objects.all().select_related("manufacturer") + result = {} + for mt in types: + result[mt.model] = mt + if mt.part_number and mt.part_number != mt.model: + result[mt.part_number] = mt + + # Overlay with explicit ModuleTypeMapping entries (take priority) + for mapping in ModuleTypeMapping.objects.select_related("netbox_module_type__manufacturer"): + result[mapping.librenms_model] = mapping.netbox_module_type + + return result + + def _find_parent_container_name(self, item, index_map): + """Resolve the parent container name for an inventory item.""" + contained_in = item.get("entPhysicalContainedIn", 0) + if contained_in == 0: + return None + parent = index_map.get(contained_in) + if parent: + return parent.get("entPhysicalName", "") + return None + + def _match_module_bay(self, item, index_map, module_bays): + """ + Try to match an inventory item to a NetBox ModuleBay. + Checks ModuleBayMapping table first (exact then regex), then falls back + to exact parent name match, then positional matching. + """ + import re + + from netbox_librenms_plugin.models import ModuleBayMapping + + parent_name = self._find_parent_container_name(item, index_map) + item_name = item.get("entPhysicalName", "") + item_descr = item.get("entPhysicalDescr", "") + phys_class = item.get("entPhysicalClass", "") + + # Build candidate names: parent, item name, item description + candidate_names = [n for n in [parent_name, item_name, item_descr] if n] + + # Check ModuleBayMapping table for each candidate (exact match) + for name in candidate_names: + filters = {"librenms_name": name, "is_regex": False} + if phys_class: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class=phys_class).first() + if not mapping: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + else: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + + if mapping and mapping.netbox_bay_name in module_bays: + return module_bays[mapping.netbox_bay_name] + + # Regex pattern matching on all candidate names + for name in candidate_names: + bay = self._lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping) + if bay and self._fpc_slot_matches(name, bay): + return bay + + # Fallback: exact match on candidate names against bay dict + for name in candidate_names: + if name in module_bays: + return module_bays[name] + + # Positional fallback: determine slot number from container sibling order + # Handles SFPs inside converters where containers are unnamed + bay = self._match_bay_by_position(item, index_map, module_bays) + if bay: + return bay + + return None + + @staticmethod + def _fpc_slot_matches(candidate_name, bay): + """Validate that a regex-matched bay's parent slot position is consistent with + a positional descriptor like 'Model @ FPC/pic/port'. + + Returns True if the descriptor has no FPC reference, or if the bay's parent + module slot position matches the FPC number in the descriptor. Prevents + orphaned top-level items (e.g. QSFP @ 1/1/1 when FPC1 is not installed) + from incorrectly matching bays belonging to a different FPC's module. + """ + import re as _re + + match = _re.search(r"@\s+(\d+)/", candidate_name) + if not match: + return True + expected_fpc = match.group(1) + module = getattr(bay, "module", None) + if not module: + return True + parent_bay = getattr(module, "module_bay", None) + if not parent_bay: + return True + return parent_bay.position == expected_fpc + + @staticmethod + def _lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping): + """Try regex ModuleBayMapping patterns against a name. + + Returns matched module bay or None. + """ + regex_filters = {"is_regex": True} + if phys_class: + regex_mappings = list(ModuleBayMapping.objects.filter(**regex_filters, librenms_class=phys_class)) + list( + ModuleBayMapping.objects.filter(**regex_filters, librenms_class="") + ) + else: + regex_mappings = list(ModuleBayMapping.objects.filter(**regex_filters, librenms_class="")) + + for mapping in regex_mappings: + try: + match = re.fullmatch(mapping.librenms_name, name) + except re.error: + continue + if match: + resolved_bay = match.expand(mapping.netbox_bay_name) + if resolved_bay in module_bays: + bay = module_bays[resolved_bay] + if BaseModuleTableView._fpc_slot_matches(name, bay): + return bay + return None + + @staticmethod + def _match_bay_by_position(item, index_map, module_bays): + """Match bay by item's positional order among container siblings. + + When an item is inside a container (no model), walk up to find the + nearest ancestor with a model, count which container slot the item + occupies, and match to the bay by number (e.g., SFP 1, SFP 2). + """ + # Walk up through modelless containers to find the parent with a model + current_idx = item.get("entPhysicalContainedIn", 0) + container_idx = None + for _ in range(5): + if not current_idx or current_idx not in index_map: + return None + ancestor = index_map[current_idx] + model = (ancestor.get("entPhysicalModelName") or "").strip() + if model: + # Found the parent with a model; container_idx is the intermediate container + break + container_idx = current_idx + current_idx = ancestor.get("entPhysicalContainedIn", 0) + else: + return None + + if not container_idx: + return None + + # Determine position: count siblings of the container under the parent + parent_with_model_idx = current_idx + siblings = sorted( + [i for i in index_map.values() if i.get("entPhysicalContainedIn") == parent_with_model_idx], + key=lambda x: x.get("entPhysicalParentRelPos", 0), + ) + slot_num = None + for i, sib in enumerate(siblings): + if sib["entPhysicalIndex"] == container_idx: + slot_num = i + 1 + break + + if slot_num is None: + return None + + # Try common bay naming patterns + for pattern in [f"SFP {slot_num}", f"Slot {slot_num}", f"Bay {slot_num}", f"Port {slot_num}"]: + if pattern in module_bays: + return module_bays[pattern] + + return None + + def _build_row(self, item, index_map, module_bays, module_types, depth=0): + """Build a single table row from a LibreNMS inventory item.""" + from netbox_librenms_plugin.utils import ( + apply_normalization_rules, + has_nested_name_conflict, + module_type_is_end_module, + module_type_uses_module_path, + module_type_uses_module_token, + supports_module_path, + ) + + model_name = item.get("entPhysicalModelName", "") or "" + serial = item.get("entPhysicalSerialNum", "") or "" + phys_class = item.get("entPhysicalClass", "") + name = item.get("entPhysicalName", "") or "-" + description = item.get("entPhysicalDescr", "") or "" + + # Match to NetBox module bay + matched_bay = self._match_module_bay(item, index_map, module_bays) + + # Match to NetBox module type (direct lookup, then normalization fallback) + matched_type = module_types.get(model_name) if model_name else None + if not matched_type and model_name: + normalized = apply_normalization_rules( + model_name, "module_type", manufacturer=getattr(self, "_device_manufacturer", None) + ) + if normalized != model_name: + matched_type = module_types.get(normalized) + + # Badge flags β€” purely informational, never block installation + needs_module_path = matched_type and module_type_uses_module_path(matched_type) + # {module_path} used but NetBox version does not support it β†’ "Upgrade NetBox" hint + netbox_upgrade_needed = bool(needs_module_path and not supports_module_path()) + # End module still using old {module} when {module_path} is available β†’ "Upgrade module-type" hint + suggest_type_upgrade = bool( + matched_type + and supports_module_path() + and module_type_is_end_module(matched_type) + and module_type_uses_module_token(matched_type) + ) + + # Check for nested module naming conflicts + name_conflict = matched_type and matched_bay and has_nested_name_conflict(matched_type, matched_bay) + + # Determine status + status = self._determine_status(matched_bay, matched_type, serial) + + row = { + "name": name, + "model": model_name or "-", + "serial": serial or "-", + "description": description, + "item_class": phys_class, + "module_bay": matched_bay.name if matched_bay else "-", + "module_type": matched_type.model if matched_type else "-", + "status": status, + "row_class": "", + "can_install": False, + "module_bay_id": matched_bay.pk if matched_bay else None, + "module_type_id": matched_type.pk if matched_type else None, + "depth": depth, + "ent_physical_index": item.get("entPhysicalIndex"), + "has_installable_children": False, + } + + if netbox_upgrade_needed: + row["row_class"] = "table-warning" + row["module_path_warning"] = ( + "This module type uses {module_path} in its interface templates. " + "The current NetBox version does not support {module_path} yet β€” " + "installation will proceed but interface naming may not work as expected. " + "Upgrade NetBox to enable full {module_path} support." + ) + + if suggest_type_upgrade: + row["module_type_upgrade_hint"] = ( + "This module type uses {module} in its interface templates. " + "Since this NetBox version supports {module_path}, consider updating " + "the module type's interface templates to use {module_path} for " + "precise per-slot interface naming." + ) + + if name_conflict: + row["row_class"] = "table-warning" + row["name_conflict_warning"] = ( + "This module type uses {module} in its interface template. " + "Installing multiple siblings will create duplicate interface names. " + "An interface naming plugin with a rewrite rule for this module type can resolve this." + ) + + # Add URLs for matched objects + if matched_bay: + row["module_bay_url"] = matched_bay.get_absolute_url() + # Check if a module is already installed in this bay + if hasattr(matched_bay, "installed_module") and matched_bay.installed_module: + installed = matched_bay.installed_module + row["installed_module"] = installed + row["module_url"] = installed.get_absolute_url() + # Check serial match + if serial and installed.serial and installed.serial.strip() == serial.strip(): + status = "Installed" + row["row_class"] = "table-success" + elif serial and installed.serial and installed.serial.strip() != serial.strip(): + status = "Serial Mismatch" + row["row_class"] = "table-danger" + else: + status = "Installed" + row["row_class"] = "table-success" + row["status"] = status + elif matched_type: + # Bay exists, type matched, no module installed β†’ can install + row["can_install"] = True + + if matched_type: + row["module_type_url"] = matched_type.get_absolute_url() + + return row + + def _determine_status(self, matched_bay, matched_type, serial): + """Determine the sync status for an inventory item.""" + if matched_bay and matched_type: + return "Matched" + if not matched_bay: + return "No Bay" + if not matched_type: + return "No Type" + return "Unmatched" + + +class InstallModuleView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, View): + """Install a NetBox Module into a ModuleBay from LibreNMS inventory data.""" + + def post(self, request, pk): + from dcim.models import Device, Module, ModuleBay, ModuleType + + self.required_object_permissions = {"POST": [("add", Module)]} + if error := self.require_all_permissions_json("POST"): + return error + + device = get_object_or_404(Device, pk=pk) + module_bay_id = request.POST.get("module_bay_id") + module_type_id = request.POST.get("module_type_id") + serial = request.POST.get("serial", "").strip() + + if not module_bay_id or not module_type_id: + messages.error(request, "Missing module bay or module type.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + module_bay = get_object_or_404(ModuleBay, pk=module_bay_id, device=device) + module_type = get_object_or_404(ModuleType, pk=module_type_id) + + # Check if bay already has a module installed + if hasattr(module_bay, "installed_module") and module_bay.installed_module: + messages.warning(request, f"Module bay '{module_bay.name}' already has a module installed.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + try: + with transaction.atomic(): + module = Module( + device=device, + module_bay=module_bay, + module_type=module_type, + serial=serial, + status="active", + ) + module.full_clean() + module.save() + + messages.success( + request, f"Installed {module_type.model} in {module_bay.name} (serial: {serial or 'N/A'})." + ) + except Exception as e: + messages.error(request, f"Failed to install module: {e}") + + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + +class InstallBranchView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, CacheMixin, View): + """Install a module and all its installable descendants from LibreNMS inventory.""" + + def post(self, request, pk): + from dcim.models import Device, Module, ModuleBay, ModuleType + + self.required_object_permissions = {"POST": [("add", Module)]} + if error := self.require_all_permissions_json("POST"): + return error + + device = get_object_or_404(Device, pk=pk) + parent_index = request.POST.get("parent_index") + + if not parent_index: + messages.error(request, "Missing parent inventory index.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + try: + parent_index = int(parent_index) + except ValueError: + messages.error(request, "Invalid parent inventory index.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Get cached inventory data + cached_data = cache.get(self.get_cache_key(device, "inventory")) + if not cached_data: + messages.error(request, "No cached inventory data. Please refresh modules first.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Build index map and collect the branch to install + index_map = {item["entPhysicalIndex"]: item for item in cached_data} + branch_items = self._collect_branch(parent_index, cached_data) + + if not branch_items: + messages.warning(request, "No installable items found in this branch.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Load module types (with mappings) + module_types = self._get_module_types() + + # Install top-down: each install may create new child bays + installed = [] + skipped = [] + failed = [] + + try: + with transaction.atomic(): + for item in branch_items: + result = self._install_single( + device, + item, + index_map, + module_types, + ModuleBay, + ModuleType, + Module, + ) + if result["status"] == "installed": + installed.append(result["name"]) + elif result["status"] == "skipped": + skipped.append(f"{result['name']}: {result['reason']}") + else: + failed.append(f"{result['name']}: {result['reason']}") + except Exception as e: + messages.error(request, f"Branch install failed: {e}") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Report results + if installed: + messages.success(request, f"Installed {len(installed)} module(s): {', '.join(installed)}") + if skipped: + messages.info(request, f"Skipped {len(skipped)}: {'; '.join(skipped)}") + if failed: + messages.warning(request, f"Failed {len(failed)}: {'; '.join(failed)}") + + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + def _collect_branch(self, parent_index, inventory_data): + """Collect all items in a branch depth-first, parent first. + + Returns items in install order (parent before children). + """ + items = [] + parent = next((i for i in inventory_data if i["entPhysicalIndex"] == parent_index), None) + if parent: + model = (parent.get("entPhysicalModelName") or "").strip() + if model: + items.append(parent) + self._collect_children(parent_index, inventory_data, items) + return items + + def _collect_children(self, parent_idx, inventory_data, items): + """Recursively collect children with models, depth-first.""" + children = [i for i in inventory_data if i.get("entPhysicalContainedIn") == parent_idx] + for child in children: + model = (child.get("entPhysicalModelName") or "").strip() + if model: + items.append(child) + # Always recurse to find deeper items (containers may lack models) + self._collect_children(child["entPhysicalIndex"], inventory_data, items) + + def _get_module_types(self): + """Get all module types indexed by model, with mappings applied.""" + from dcim.models import ModuleType + + from netbox_librenms_plugin.models import ModuleTypeMapping + + types = ModuleType.objects.all().select_related("manufacturer") + result = {} + for mt in types: + result[mt.model] = mt + if mt.part_number and mt.part_number != mt.model: + result[mt.part_number] = mt + for mapping in ModuleTypeMapping.objects.select_related("netbox_module_type__manufacturer"): + result[mapping.librenms_model] = mapping.netbox_module_type + return result + + def _install_single(self, device, item, index_map, module_types, ModuleBay, ModuleType, Module): + """Try to install a single inventory item. + + Re-fetches module bays each time since parent installs create new ones. + Scopes bay lookup to the correct parent module to handle duplicate bay names. + """ + from netbox_librenms_plugin.models import ModuleBayMapping + from netbox_librenms_plugin.utils import apply_normalization_rules + + model_name = (item.get("entPhysicalModelName") or "").strip() + serial = (item.get("entPhysicalSerialNum") or "").strip() + name = item.get("entPhysicalName", "") or model_name + + # Match module type (direct, then normalization fallback) + matched_type = module_types.get(model_name) + if not matched_type and model_name: + manufacturer = getattr(getattr(device, "device_type", None), "manufacturer", None) + normalized = apply_normalization_rules(model_name, "module_type", manufacturer=manufacturer) + if normalized != model_name: + matched_type = module_types.get(normalized) + if not matched_type: + return {"status": "skipped", "name": name, "reason": "no matching type"} + + # Re-fetch module bays (parent install creates new child bays) + bays = ModuleBay.objects.filter(device=device).select_related("installed_module__module_type") + + # Determine if this item belongs under an installed module + # by tracing its LibreNMS parent hierarchy to an installed item + parent_module_id = self._find_parent_module_id(item, index_map, device, ModuleBay) + + if parent_module_id: + bay_dict = {bay.name: bay for bay in bays if bay.module_id == parent_module_id} + else: + bay_dict = {bay.name: bay for bay in bays if not bay.module_id} + + # Match module bay using mapping table + matched_bay = self._match_bay(item, index_map, bay_dict, ModuleBayMapping) + if not matched_bay: + return {"status": "skipped", "name": name, "reason": "no matching bay"} + + # Check if already installed + if hasattr(matched_bay, "installed_module") and matched_bay.installed_module: + return {"status": "skipped", "name": name, "reason": "bay already occupied"} + + # Install + try: + with transaction.atomic(): # savepoint: failure here won't abort parent tx + module = Module( + device=device, + module_bay=matched_bay, + module_type=matched_type, + serial=serial, + status="active", + ) + module.full_clean() + module.save() + except Exception as e: + error_msg = str(e) + if "dcim_interface_unique_device_name" in error_msg: + error_msg = ( + "duplicate interface name β€” this module type's interface template " + "uses {module} which resolves to the same name for all siblings. " + "An interface naming plugin with a rewrite rule for this module type can fix this." + ) + return {"status": "failed", "name": name, "reason": error_msg} + + return {"status": "installed", "name": f"{matched_type.model} β†’ {matched_bay.name}"} + + @staticmethod + def _find_parent_module_id(item, index_map, device, ModuleBay): + """Find the NetBox module ID for the installed parent of this inventory item. + + Walks up the LibreNMS hierarchy to find an ancestor whose name matches + an installed module bay on the device. + """ + from netbox_librenms_plugin.models import ModuleBayMapping + + current = item + for _ in range(10): # max depth guard + parent_idx = current.get("entPhysicalContainedIn", 0) + if not parent_idx or parent_idx not in index_map: + return None + parent = index_map[parent_idx] + parent_name = parent.get("entPhysicalName", "") + parent_descr = parent.get("entPhysicalDescr", "") + + # Check if this parent matches an installed module bay on the device + device_bays = ModuleBay.objects.filter(device=device, module_id__isnull=True).select_related( + "installed_module" + ) + + for bay in device_bays: + if hasattr(bay, "installed_module") and bay.installed_module: + if bay.name == parent_name or (parent_descr and bay.name == parent_descr): + return bay.installed_module.pk + + # Also check ModuleBayMapping for indirect matches + for name in [parent_name, parent_descr]: + if not name: + continue + mapping = ModuleBayMapping.objects.filter(librenms_name=name).first() + if mapping: + bay = ( + ModuleBay.objects.filter(device=device, name=mapping.netbox_bay_name, module_id__isnull=True) + .select_related("installed_module") + .first() + ) + if bay and hasattr(bay, "installed_module") and bay.installed_module: + return bay.installed_module.pk + + current = parent + return None + + @staticmethod + def _match_bay(item, index_map, module_bays, ModuleBayMapping): + """Match an inventory item to a module bay (same logic as BaseModuleTableView).""" + import re + + # Resolve parent name + contained_in = item.get("entPhysicalContainedIn", 0) + parent_name = None + if contained_in: + parent = index_map.get(contained_in) + if parent: + parent_name = parent.get("entPhysicalName", "") + + item_name = item.get("entPhysicalName", "") + item_descr = item.get("entPhysicalDescr", "") + phys_class = item.get("entPhysicalClass", "") + + # Build candidate names: parent, item name, item description + candidate_names = [n for n in [parent_name, item_name, item_descr] if n] + + # Check mapping for each candidate (exact match) + for name in candidate_names: + filters = {"librenms_name": name, "is_regex": False} + if phys_class: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class=phys_class).first() + if not mapping: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + else: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + if mapping and mapping.netbox_bay_name in module_bays: + return module_bays[mapping.netbox_bay_name] + + # Regex pattern matching on all candidate names + for name in candidate_names: + bay = BaseModuleTableView._lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping) + if bay: + return bay + + # Fallback: exact match on candidate names against bay dict + for name in candidate_names: + if name in module_bays: + return module_bays[name] + + # Positional fallback for items inside converters + return BaseModuleTableView._match_bay_by_position(item, index_map, module_bays) diff --git a/netbox_librenms_plugin/views/imports/__init__.py b/netbox_librenms_plugin/views/imports/__init__.py index b6d22d0e1d..3a46ab3625 100644 --- a/netbox_librenms_plugin/views/imports/__init__.py +++ b/netbox_librenms_plugin/views/imports/__init__.py @@ -4,6 +4,7 @@ BulkImportConfirmView, BulkImportDevicesView, DeviceClusterUpdateView, + DeviceConflictActionView, DeviceRackUpdateView, DeviceRoleUpdateView, DeviceValidationDetailsView, @@ -16,6 +17,7 @@ "BulkImportConfirmView", "BulkImportDevicesView", "DeviceClusterUpdateView", + "DeviceConflictActionView", "DeviceRackUpdateView", "DeviceRoleUpdateView", "DeviceValidationDetailsView", diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 46b10b117a..61b0110a05 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -5,9 +5,11 @@ from django.contrib import messages from django.core.cache import cache -from django.core.exceptions import PermissionDenied +from django.core.exceptions import PermissionDenied, ValidationError +from django.db import transaction from django.http import HttpResponse, JsonResponse from django.shortcuts import redirect, render +from django.utils.html import escape from django.views import View from netbox_librenms_plugin.import_utils import ( @@ -29,11 +31,94 @@ fetch_model_by_id, ) from netbox_librenms_plugin.tables.device_status import DeviceImportTable -from netbox_librenms_plugin.utils import save_user_pref -from netbox_librenms_plugin.views.mixins import LibreNMSAPIMixin, LibreNMSPermissionMixin +from netbox_librenms_plugin.utils import get_user_pref, save_user_pref, set_librenms_device_id +from netbox_librenms_plugin.views.mixins import LibreNMSAPIMixin, LibreNMSPermissionMixin, NetBoxObjectPermissionMixin logger = logging.getLogger(__name__) +# Actions that require the force checkbox when a device-type mismatch is detected. +_FORCE_REQUIRED_ACTIONS = frozenset({"link", "update", "update_serial", "update_type"}) + + +def _resolve_naming_preferences(request) -> tuple[bool, bool]: + """Resolve use_sysname/strip_domain: POST/GET data β†’ user pref β†’ plugin settings.""" + from netbox_librenms_plugin.models import LibreNMSSettings + + settings = None + + # Check POST first (form submissions), then GET (HTMX hx-include on hx-get). + # Support hyphenated ("use-sysname-toggle"), underscored ("use_sysname-toggle"), + # and plain canonical ("use_sysname") key variants for compatibility across + # different form/hidden-input implementations. + _USE_SYSNAME_KEYS = ("use-sysname-toggle", "use_sysname-toggle", "use_sysname") + _STRIP_DOMAIN_KEYS = ("strip-domain-toggle", "strip_domain-toggle", "strip_domain") + _TRUTHY = frozenset({"on", "true", "1"}) + + def _is_truthy(val): + return val.lower() in _TRUTHY if val is not None else False + + _use_sysname_post = next((request.POST.get(k) for k in _USE_SYSNAME_KEYS if k in request.POST), None) + _use_sysname_get = next((request.GET.get(k) for k in _USE_SYSNAME_KEYS if k in request.GET), None) + + if _use_sysname_post is not None: + use_sysname = _is_truthy(_use_sysname_post) + elif _use_sysname_get is not None: + use_sysname = _is_truthy(_use_sysname_get) + else: + pref = get_user_pref(request, "plugins.netbox_librenms_plugin.use_sysname") + if pref is not None: + use_sysname = pref + else: + settings = LibreNMSSettings.objects.first() + use_sysname = getattr(settings, "use_sysname_default", True) if settings else True + + _strip_domain_post = next((request.POST.get(k) for k in _STRIP_DOMAIN_KEYS if k in request.POST), None) + _strip_domain_get = next((request.GET.get(k) for k in _STRIP_DOMAIN_KEYS if k in request.GET), None) + + if _strip_domain_post is not None: + strip_domain = _is_truthy(_strip_domain_post) + elif _strip_domain_get is not None: + strip_domain = _is_truthy(_strip_domain_get) + else: + pref = get_user_pref(request, "plugins.netbox_librenms_plugin.strip_domain") + if pref is not None: + strip_domain = pref + else: + if settings is None: + settings = LibreNMSSettings.objects.first() + strip_domain = getattr(settings, "strip_domain_default", False) if settings else False + + return use_sysname, strip_domain + + +def _get_hostname_for_action(request, validation: dict, libre_device: dict) -> str: + """Return the resolved hostname to use when updating a device during a conflict action. + + Prefer the cached ``resolved_name`` from validation (already computed with the + user's naming prefs at validation time). Fall back to computing it fresh from + the current request's naming preferences. + """ + resolved = validation.get("resolved_name") + if resolved: + return resolved + use_sysname, strip_domain = _resolve_naming_preferences(request) + return _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) + + +def _save_device(device) -> HttpResponse | None: + """Call full_clean() then save(). Return an HttpResponse on failure, None on success.""" + from django.db import IntegrityError + + try: + device.full_clean() + except ValidationError as exc: + return HttpResponse(f"Validation error: {escape(str(exc))}", status=400) + try: + device.save() + except IntegrityError as exc: + return HttpResponse(f"Integrity error: {escape(str(exc))}", status=409) + return None + class DeviceImportHelperMixin: """Mixin providing common validation and rendering helpers for device import views.""" @@ -113,11 +198,17 @@ def get_validated_device_with_selections(self, device_id: int, request) -> tuple # This checks: user preference, cache status, and VM vs Device enable_vc = not is_vm and self._should_enable_vc_detection(device_id, request) + # Extract naming preferences: POST data (hx-include) β†’ user pref β†’ plugin settings. + use_sysname, strip_domain = _resolve_naming_preferences(request) + validation = validate_device_for_import( libre_device, import_as_vm=is_vm, api=self.librenms_api if enable_vc else None, include_vc_detection=enable_vc, + use_sysname=use_sysname, + strip_domain=strip_domain, + server_key=self.librenms_api.server_key, ) validation["import_as_vm"] = is_vm @@ -223,8 +314,7 @@ def post(self, request): status=400, ) - use_sysname = request.POST.get("use-sysname-toggle") == "on" - strip_domain = request.POST.get("strip-domain-toggle") == "on" + use_sysname, strip_domain = _resolve_naming_preferences(request) devices = [] errors = [] @@ -259,19 +349,21 @@ def post(self, request): rack_id = selections["rack_id"] is_vm = bool(cluster_id) - validation = validate_device_for_import(libre_device, import_as_vm=is_vm, api=self.librenms_api) + validation = validate_device_for_import( + libre_device, + import_as_vm=is_vm, + api=self.librenms_api, + use_sysname=use_sysname, + strip_domain=strip_domain, + server_key=self.librenms_api.server_key, + ) # Mark validation with VC detection flag for proper URL generation in table # Bulk confirm should respect the initial filter's VC detection preference vc_requested = request.GET.get("enable_vc_detection") == "true" validation["_vc_detection_enabled"] = vc_requested - device_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=strip_domain, - device_id=device_id, - ) + device_name = validation.get("resolved_name") if validation.get("virtual_chassis", {}).get("is_stack") and device_name: validation["virtual_chassis"] = update_vc_member_suggested_names( @@ -624,11 +716,16 @@ def post(self, request): # noqa: PLR0912 - branching keeps responses explicit is_vm = device_id in [item["device_id"] for item in vm_result.get("success", [])] # Re-validate with fresh status (will now show as imported) + # Pass naming preferences so name comparison uses the same + # resolved name the device was imported with. validation = validate_device_for_import( libre_device, import_as_vm=is_vm, api=None, # No VC detection needed for already-imported devices include_vc_detection=False, + server_key=self.librenms_api.server_key, + use_sysname=sync_options.get("use_sysname", True), + strip_domain=sync_options.get("strip_domain", False), ) validation["import_as_vm"] = is_vm @@ -702,17 +799,108 @@ def get(self, request, device_id): status=404, ) + use_sysname, strip_domain = _resolve_naming_preferences(request) + context = { "libre_device": libre_device, "validation": validation, + "use_sysname": use_sysname, + "strip_domain": strip_domain, } + # Add sync comparison data for existing devices + existing = validation.get("existing_device") + if existing: + context["sync_info"] = self._build_sync_info(libre_device, existing) + context["existing_id_servers"] = self._build_id_server_info(existing) + return render( request, "netbox_librenms_plugin/htmx/device_validation_details.html", context, ) + @staticmethod + def _build_sync_info(libre_device, existing_device): + """Build sync comparison data between LibreNMS device and existing NetBox device.""" + librenms_serial = libre_device.get("serial") or "-" + librenms_os = libre_device.get("os") or "-" + librenms_hardware = libre_device.get("hardware") or "-" + + # Serial comparison + serial_synced = existing_device.serial == librenms_serial or librenms_serial == "-" + + # Platform comparison + platform_info = { + "netbox_platform": getattr(existing_device, "platform", None), + "librenms_os": librenms_os, + "platform_exists": False, + "matching_platform": None, + } + if librenms_os and librenms_os != "-": + from netbox_librenms_plugin.utils import find_matching_platform + + match_result = find_matching_platform(librenms_os) + if match_result["found"]: + platform_info["platform_exists"] = True + platform_info["matching_platform"] = match_result["platform"] + + netbox_platform = platform_info["netbox_platform"] + matching_platform = platform_info["matching_platform"] + platform_synced = librenms_os == "-" or bool( + netbox_platform and matching_platform and netbox_platform.pk == matching_platform.pk + ) + + # Device type comparison + device_type_synced = True + librenms_device_type = None + if librenms_hardware and librenms_hardware != "-": + from netbox_librenms_plugin.utils import match_librenms_hardware_to_device_type + + hw_match = match_librenms_hardware_to_device_type(librenms_hardware) + if hw_match.get("matched"): + librenms_device_type = hw_match["device_type"] + if not existing_device.device_type or existing_device.device_type.pk != librenms_device_type.pk: + device_type_synced = False + else: + device_type_synced = False + + all_synced = serial_synced and platform_synced and device_type_synced + + return { + "librenms_serial": librenms_serial, + "serial_synced": serial_synced, + "platform_info": platform_info, + "platform_synced": platform_synced, + "librenms_hardware": librenms_hardware, + "librenms_device_type": librenms_device_type, + "device_type_synced": device_type_synced, + "all_synced": all_synced, + } + + @staticmethod + def _build_id_server_info(existing_device): + """Return per-server ID mappings for the existing device's librenms_id custom field. + + Returns a list of dicts with server_key, display_name, and device_id β€” one entry + per server the device is linked to. Returns None when the format is legacy (bare int) + or when the field is absent/invalid. + """ + from django.conf import settings + + cf_value = existing_device.custom_field_data.get("librenms_id") + if not isinstance(cf_value, dict): + return None + + plugins_config = settings.PLUGINS_CONFIG.get("netbox_librenms_plugin", {}) + servers_config = plugins_config.get("servers", {}) + result = [] + for sk, did in cf_value.items(): + srv_cfg = servers_config.get(sk, {}) + display_name = srv_cfg.get("display_name") or sk + result.append({"server_key": sk, "display_name": display_name, "device_id": did}) + return result or None + class DeviceRoleUpdateView(LibreNMSPermissionMixin, LibreNMSAPIMixin, DeviceImportHelperMixin, View): """HTMX view to update a table row when a role is selected.""" @@ -753,6 +941,312 @@ def post(self, request, device_id): return self.render_device_row(request, libre_device, validation, selections) +class DeviceConflictActionView( + LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, LibreNMSAPIMixin, DeviceImportHelperMixin, View +): + """HTMX view to resolve device conflicts (link, update, update serial).""" + + def post(self, request, device_id): + """Resolve a device conflict by linking, updating, or syncing serial.""" + if error := self.require_write_permission(): + return error + + from dcim.models import Device + + action = request.POST.get("action") + existing_device_id = request.POST.get("existing_device_id") + + if not action or not existing_device_id: + return HttpResponse("Missing action or existing_device_id", status=400) + + try: + existing_device = Device.objects.get(pk=int(existing_device_id)) + except (Device.DoesNotExist, ValueError): + return HttpResponse("Existing device not found", status=404) + + # Object-level change permission for the specific device being mutated. + self.required_object_permissions = {"POST": [("change", Device)]} + if error := self.require_object_permissions("POST"): + return error + + libre_device, validation, selections = self.get_validated_device_with_selections(device_id, request) + if not libre_device: + return HttpResponse("LibreNMS device not found", status=404) + + # Verify the POSTed existing_device_id matches the validated conflict target. + # Require a confirmed conflict target: if validation has no existing_device, the + # LibreNMS device was not validated against this NetBox device, so mutations are unsafe. + validated_existing = validation.get("existing_device") if validation else None + if validated_existing is None: + return HttpResponse("Missing validated conflict target", status=400) + if validated_existing.pk != existing_device.pk: + return HttpResponse("Device ID mismatch: existing_device_id does not match validated device", status=400) + + # Require force flag when device type mismatches, but only for actions that use it + force = request.POST.get("force") == "on" + if validation.get("device_type_mismatch") and action in _FORCE_REQUIRED_ACTIONS and not force: + return HttpResponse( + "Device type mismatch detected. Check the force checkbox to proceed.", + status=400, + ) + + # When force is used with device_type_mismatch, update device type to LibreNMS value + librenms_device_type = None + if validation.get("device_type_mismatch") and force: + librenms_device_type = validation.get("device_type", {}).get("device_type") + + librenms_id = libre_device.get("device_id") + try: + librenms_id = int(librenms_id) + except (TypeError, ValueError): + return HttpResponse("Invalid or missing LibreNMS device_id in payload", status=400) + + # Wrap the LibreNMS-ID collision check and subsequent write in a single + # transaction so the read-then-write is atomic for link/update/update_serial. + # NOTE: A fully race-free guarantee would require a DB-unique constraint on + # (server_key, librenms_id) β€” e.g., a dedicated DeviceLibreNMSIDMapping model. + # That is deferred to a future schema migration. Until then, we acquire a + # row-level lock on the target device before re-checking for conflicts, which + # serializes concurrent operations on the SAME device and greatly reduces the + # window for assigning the same ID to two DIFFERENT devices. + if action in {"link", "update", "update_serial"}: + from django.db.models import Q + + with transaction.atomic(): + server_key = self.librenms_api.server_key + # Lock the target device row so concurrent requests for the same + # device are serialized. The conflict check below is still a + # best-effort guard for different devices; a DB unique constraint + # would be needed for full protection. + try: + existing_device = Device.objects.select_for_update().get(pk=existing_device.pk) + except Device.DoesNotExist: + return HttpResponse( + "Device no longer exists; it may have been deleted concurrently.", + status=409, + ) + conflict_exists = ( + Device.objects.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}) + | Q(custom_field_data__librenms_id=librenms_id) + ) + .exclude(pk=existing_device.pk) + .exists() + ) + if conflict_exists: + return HttpResponse( + f"LibreNMS ID conflict: ID {librenms_id} is already assigned to another device.", + status=409, + ) + + if action == "link": + # Link to LibreNMS and update name from LibreNMS data + hostname = _get_hostname_for_action(request, validation, libre_device) + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) + existing_device.name = hostname + if librenms_device_type: + existing_device.device_type = librenms_device_type + if err := _save_device(existing_device): + return err + logger.info(f"Linked device '{existing_device.name}' to LibreNMS ID {librenms_id}") + + elif action == "update": + # Update hostname, serial, and link to LibreNMS + hostname = _get_hostname_for_action(request, validation, libre_device) + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-": + # Lock any conflicting device under the same transaction to reduce + # the serial-assignment race window (best-effort; a DB unique + # constraint on serial would give full protection). + conflict_device = ( + Device.objects.select_for_update() + .filter(serial=incoming_serial) + .exclude(pk=existing_device.pk) + .first() + ) + if conflict_device: + return HttpResponse( + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", + status=409, + ) + existing_device.serial = incoming_serial + existing_device.name = hostname + if librenms_device_type: + existing_device.device_type = librenms_device_type + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) + if err := _save_device(existing_device): + return err + logger.info( + f"Updated device '{existing_device.name}': serial={incoming_serial}, " + f"linked to LibreNMS ID {librenms_id}" + ) + + elif action == "update_serial": + # Update only the serial and link to LibreNMS + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-": + # Lock any conflicting device under the same transaction to reduce + # the serial-assignment race window (best-effort; a DB unique + # constraint on serial would give full protection). + conflict_device = ( + Device.objects.select_for_update() + .filter(serial=incoming_serial) + .exclude(pk=existing_device.pk) + .first() + ) + if conflict_device: + return HttpResponse( + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", + status=409, + ) + existing_device.serial = incoming_serial + if librenms_device_type: + existing_device.device_type = librenms_device_type + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) + if err := _save_device(existing_device): + return err + logger.info( + f"Updated serial on device '{existing_device.name}' to {incoming_serial}, " + f"linked to LibreNMS ID {librenms_id}" + ) + + elif action == "sync_name": + # Sync device name from LibreNMS (e.g., IP β†’ sysName) + hostname = _get_hostname_for_action(request, validation, libre_device) + existing_device.name = hostname + if err := _save_device(existing_device): + return err + logger.info(f"Synced name on device '{existing_device.name}' from LibreNMS") + + elif action == "update_type": + # Update device type from LibreNMS (requires force for mismatch) + if librenms_device_type: + existing_device.device_type = librenms_device_type + if err := _save_device(existing_device): + return err + logger.info(f"Updated device type on '{existing_device.name}' to {librenms_device_type}") + else: + return HttpResponse("No LibreNMS device type available to update", status=400) + + elif action == "sync_serial": + # Sync serial number from LibreNMS. + # Wrap conflict-check-and-write in a transaction with a row lock so + # concurrent requests cannot both pass the serial uniqueness guard. + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-": + with transaction.atomic(): + try: + locked_device = Device.objects.select_for_update().get(pk=existing_device.pk) + except Device.DoesNotExist: + return HttpResponse( + "Device no longer exists; it may have been deleted concurrently.", + status=409, + ) + # Re-check for serial ownership conflict under lock + conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=locked_device.pk).first() + if conflict_device: + logger.warning( + f"Serial sync blocked: '{incoming_serial}' already assigned to " + f"'{conflict_device.name}' (pk={conflict_device.pk})" + ) + return HttpResponse( + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", + status=409, + ) + locked_device.serial = incoming_serial + if err := _save_device(locked_device): + return err + logger.info(f"Synced serial on '{locked_device.name}' to {incoming_serial}") + else: + return HttpResponse("No valid serial from LibreNMS", status=400) + + elif action == "sync_platform": + # Sync platform from LibreNMS OS + from netbox_librenms_plugin.utils import find_matching_platform + + librenms_os = libre_device.get("os") or "" + if librenms_os and librenms_os != "-": + match_result = find_matching_platform(librenms_os) + if match_result["found"]: + existing_device.platform = match_result["platform"] + if err := _save_device(existing_device): + return err + logger.info(f"Synced platform on '{existing_device.name}' to {match_result['platform']}") + else: + return HttpResponse(f"Platform '{escape(librenms_os)}' not found in NetBox", status=400) + else: + return HttpResponse("No OS info from LibreNMS", status=400) + + elif action == "sync_device_type": + # Sync device type from LibreNMS hardware (non-mismatch case) + from netbox_librenms_plugin.utils import match_librenms_hardware_to_device_type + + hardware = libre_device.get("hardware") or "" + hw_match = match_librenms_hardware_to_device_type(hardware) + if hw_match.get("matched"): + existing_device.device_type = hw_match["device_type"] + if err := _save_device(existing_device): + return err + logger.info(f"Synced device type on '{existing_device.name}' to {hw_match['device_type']}") + else: + return HttpResponse(f"No matching device type for '{escape(hardware)}'", status=400) + + elif action == "migrate_librenms_id": + # Migrate legacy bare-integer librenms_id to the JSON dict format. + # Only safe when the integer matches the LibreNMS device ID for this server, + # confirmed by serial match (or explicit force). + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + # Direct access needed to detect legacy integer format for migration prompt: + # LibreNMSAPI.get_librenms_id() returns an int in both formats; only the raw + # type check on custom_field_data reveals whether migration is needed. + cf_value = existing_device.custom_field_data.get("librenms_id") + if not isinstance(cf_value, int): + return HttpResponse( + "Device librenms_id is already in JSON format; no migration needed.", + status=400, + ) + # Verify the stored legacy ID matches the active LibreNMS device_id so we don't + # migrate a stale/incorrect association to the wrong server mapping. + if cf_value != librenms_id: + return HttpResponse( + f"Legacy librenms_id ({cf_value}) does not match the active device ID " + f"({librenms_id}); cannot migrate safely.", + status=400, + ) + if not validation.get("serial_confirmed") and not force: + return HttpResponse( + "Serial number not confirmed. Check the force checkbox to migrate without serial verification.", + status=400, + ) + migrate_legacy_librenms_id(existing_device, self.librenms_api.server_key) + if err := _save_device(existing_device): + return err + logger.info( + f"Migrated legacy librenms_id on '{existing_device.name}' " + f"to {{{self.librenms_api.server_key!r}: {cf_value}}}" + ) + + else: + return HttpResponse(f"Unknown action: {escape(action)}", status=400) + + # Clear cached validation so re-validation picks up the changes + cache_key = get_import_device_cache_key(device_id, self.librenms_api.server_key) + cache.delete(cache_key) + + # Re-validate and render updated row + libre_device, validation, selections = self.get_validated_device_with_selections(device_id, request) + if not libre_device: + return HttpResponse("Device not found after action", status=404) + + response = self.render_device_row(request, libre_device, validation, selections) + response["HX-Trigger"] = "closeModal" + return response + + class SaveUserPrefView(LibreNMSPermissionMixin, View): """Save a user preference via POST. Used by JS toggle handlers.""" diff --git a/netbox_librenms_plugin/views/imports/list.py b/netbox_librenms_plugin/views/imports/list.py index b6222a5905..c8b00c5ef1 100644 --- a/netbox_librenms_plugin/views/imports/list.py +++ b/netbox_librenms_plugin/views/imports/list.py @@ -92,6 +92,8 @@ def _load_job_results(self, job_id): filters = job_data.get("filters", {}) server_key = job_data.get("server_key", "default") vc_enabled = job_data.get("vc_detection_enabled", False) + use_sysname = job_data.get("use_sysname", True) + strip_domain = job_data.get("strip_domain", False) # Extract cache metadata for frontend warnings self._cache_timestamp = job_data.get("cached_at") @@ -109,6 +111,8 @@ def _load_job_results(self, job_id): filters=filters, device_id=device_id, vc_enabled=vc_enabled, + use_sysname=use_sysname, + strip_domain=strip_domain, ) device = cache.get(cache_key) if device: @@ -236,9 +240,9 @@ def get(self, request, *args, **kwargs): # noqa: D401 - inherited doc return_cache_status=True, ) devices_cached = devices_from_cache - except Exception: + except Exception as e: # Cache check failed; proceed with background job decision based on device_count - pass + logger.debug("Cache check failed; proceeding without cached result: %s", e, exc_info=True) # Get device count for background job decision try: @@ -252,8 +256,29 @@ def get(self, request, *args, **kwargs): # noqa: D401 - inherited doc logger.error(f"Error getting device count: {e}") device_count = 0 - # Load settings for background job decision + # Load settings for background job decision; resolve naming preferences. + # We intentionally read user_pref here rather than request.GET because the + # naming toggles (use-sysname-toggle, strip-domain-toggle) live OUTSIDE the + # filter form (method="get") and are not submitted with it. Instead, each + # toggle fires a savePref() AJAX call on change, so the user_pref is always + # up-to-date by the time the filter form is submitted. settings = None + try: + settings = LibreNMSSettings.objects.first() + except Exception: + logger.exception("Failed to load LibreNMSSettings for background job naming prefs") + _use_sysname_pref = get_user_pref(request, "plugins.netbox_librenms_plugin.use_sysname") + _strip_domain_pref = get_user_pref(request, "plugins.netbox_librenms_plugin.strip_domain") + _use_sysname = ( + _use_sysname_pref + if _use_sysname_pref is not None + else (getattr(settings, "use_sysname_default", True) if settings else True) + ) + _strip_domain = ( + _strip_domain_pref + if _strip_domain_pref is not None + else (getattr(settings, "strip_domain_default", False) if settings else False) + ) # Decide whether to use background job # Skip background job if data is already cached if not devices_cached and self.should_use_background_job(): @@ -270,6 +295,8 @@ def get(self, request, *args, **kwargs): # noqa: D401 - inherited doc show_disabled=bool(self._filter_form_data.get("show_disabled")), exclude_existing=bool(self._filter_form_data.get("exclude_existing")), server_key=self.librenms_api.server_key, + use_sysname=_use_sysname, + strip_domain=_strip_domain, ) logger.info( @@ -305,6 +332,11 @@ def get(self, request, *args, **kwargs): # noqa: D401 - inherited doc try: settings, _ = LibreNMSSettings.objects.get_or_create() except Exception: + _user = getattr(request, "user", None) + logger.exception( + "Failed to get or create LibreNMSSettings during LibreNMS import for user %s", + getattr(_user, "username", str(_user)), + ) settings = None # User preference overrides for toggles (persisted per-user) @@ -408,6 +440,32 @@ def _get_import_queryset(self): show_disabled = bool(data_source.get("show_disabled")) exclude_existing = bool(data_source.get("exclude_existing")) + # Resolve naming preferences: submitted form toggle β†’ user pref β†’ settings default. + # When pref is None (first-time user) any explicit toggle in data_source should still win. + use_sysname_pref = get_user_pref(self._request, "plugins.netbox_librenms_plugin.use_sysname") + strip_domain_pref = get_user_pref(self._request, "plugins.netbox_librenms_plugin.strip_domain") + try: + _settings = LibreNMSSettings.objects.first() + except Exception: + logger.exception("Failed to load LibreNMSSettings for naming preferences") + _settings = None + _use_sysname_toggle = data_source.get("use_sysname_toggle") + use_sysname = ( + _use_sysname_toggle + if _use_sysname_toggle is not None + else use_sysname_pref + if use_sysname_pref is not None + else (getattr(_settings, "use_sysname_default", True) if _settings else True) + ) + _strip_domain_toggle = data_source.get("strip_domain_toggle") + strip_domain = ( + _strip_domain_toggle + if _strip_domain_toggle is not None + else strip_domain_pref + if strip_domain_pref is not None + else (getattr(_settings, "strip_domain_default", False) if _settings else False) + ) + validated_devices, from_cache = process_device_filters( api=self.librenms_api, filters=libre_filters, @@ -417,6 +475,8 @@ def _get_import_queryset(self): exclude_existing=exclude_existing, request=self._request, return_cache_status=True, + use_sysname=use_sysname, + strip_domain=strip_domain, ) self._from_cache = from_cache @@ -430,6 +490,8 @@ def _get_import_queryset(self): server_key=self.librenms_api.server_key, filters=libre_filters, vc_enabled=vc_detection_enabled, + use_sysname=use_sysname, + strip_domain=strip_domain, ) cache_metadata = cache.get(cache_metadata_key) if cache_metadata: diff --git a/netbox_librenms_plugin/views/mapping_views.py b/netbox_librenms_plugin/views/mapping_views.py index b1fcec9c77..55ff7bd658 100644 --- a/netbox_librenms_plugin/views/mapping_views.py +++ b/netbox_librenms_plugin/views/mapping_views.py @@ -1,14 +1,44 @@ from netbox.views import generic from utilities.views import register_model_view -from netbox_librenms_plugin.filters import InterfaceTypeMappingFilterSet +from netbox_librenms_plugin.filters import ( + DeviceTypeMappingFilterSet, + InterfaceTypeMappingFilterSet, + ModuleBayMappingFilterSet, + ModuleTypeMappingFilterSet, + NormalizationRuleFilterSet, +) from netbox_librenms_plugin.forms import ( + DeviceTypeMappingFilterForm, + DeviceTypeMappingForm, + DeviceTypeMappingImportForm, InterfaceTypeMappingFilterForm, InterfaceTypeMappingForm, InterfaceTypeMappingImportForm, + ModuleBayMappingFilterForm, + ModuleBayMappingForm, + ModuleBayMappingImportForm, + ModuleTypeMappingFilterForm, + ModuleTypeMappingForm, + ModuleTypeMappingImportForm, + NormalizationRuleFilterForm, + NormalizationRuleForm, + NormalizationRuleImportForm, +) +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) +from netbox_librenms_plugin.tables.mappings import ( + DeviceTypeMappingTable, + InterfaceTypeMappingTable, + ModuleBayMappingTable, + ModuleTypeMappingTable, + NormalizationRuleTable, ) -from netbox_librenms_plugin.models import InterfaceTypeMapping -from netbox_librenms_plugin.tables.mappings import InterfaceTypeMappingTable from netbox_librenms_plugin.views.mixins import LibreNMSPermissionMixin @@ -84,3 +114,243 @@ class InterfaceTypeMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectC """ queryset = InterfaceTypeMapping.objects.all() + + +# --- DeviceTypeMapping views --- + + +class DeviceTypeMappingListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all DeviceTypeMapping objects.""" + + queryset = DeviceTypeMapping.objects.all() + table = DeviceTypeMappingTable + filterset = DeviceTypeMappingFilterSet + filterset_form = DeviceTypeMappingFilterForm + template_name = "netbox_librenms_plugin/devicetypemapping_list.html" + + +class DeviceTypeMappingCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + form = DeviceTypeMappingForm + + +@register_model_view(DeviceTypeMapping, "bulk_import", path="import", detail=False) +class DeviceTypeMappingBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing DeviceTypeMapping objects.""" + + queryset = DeviceTypeMapping.objects.all() + model_form = DeviceTypeMappingImportForm + + +class DeviceTypeMappingView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + + +class DeviceTypeMappingEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + form = DeviceTypeMappingForm + + +class DeviceTypeMappingDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + + +class DeviceTypeMappingBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple DeviceTypeMapping objects.""" + + queryset = DeviceTypeMapping.objects.all() + table = DeviceTypeMappingTable + + +class DeviceTypeMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + + +# --- ModuleTypeMapping views --- + + +class ModuleTypeMappingListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all ModuleTypeMapping objects.""" + + queryset = ModuleTypeMapping.objects.all() + table = ModuleTypeMappingTable + filterset = ModuleTypeMappingFilterSet + filterset_form = ModuleTypeMappingFilterForm + template_name = "netbox_librenms_plugin/moduletypemapping_list.html" + + +class ModuleTypeMappingCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + form = ModuleTypeMappingForm + + +@register_model_view(ModuleTypeMapping, "bulk_import", path="import", detail=False) +class ModuleTypeMappingBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing ModuleTypeMapping objects.""" + + queryset = ModuleTypeMapping.objects.all() + model_form = ModuleTypeMappingImportForm + + +class ModuleTypeMappingView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + + +class ModuleTypeMappingEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + form = ModuleTypeMappingForm + + +class ModuleTypeMappingDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + + +class ModuleTypeMappingBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple ModuleTypeMapping objects.""" + + queryset = ModuleTypeMapping.objects.all() + table = ModuleTypeMappingTable + + +class ModuleTypeMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + + +# --- ModuleBayMapping views --- + + +class ModuleBayMappingListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all ModuleBayMapping objects.""" + + queryset = ModuleBayMapping.objects.all() + table = ModuleBayMappingTable + filterset = ModuleBayMappingFilterSet + filterset_form = ModuleBayMappingFilterForm + template_name = "netbox_librenms_plugin/modulebaymapping_list.html" + + +class ModuleBayMappingCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + form = ModuleBayMappingForm + + +@register_model_view(ModuleBayMapping, "bulk_import", path="import", detail=False) +class ModuleBayMappingBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing ModuleBayMapping objects.""" + + queryset = ModuleBayMapping.objects.all() + model_form = ModuleBayMappingImportForm + + +class ModuleBayMappingView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + + +class ModuleBayMappingEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + form = ModuleBayMappingForm + + +class ModuleBayMappingDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + + +class ModuleBayMappingBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple ModuleBayMapping objects.""" + + queryset = ModuleBayMapping.objects.all() + table = ModuleBayMappingTable + + +class ModuleBayMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + + +# --- NormalizationRule views --- + + +class NormalizationRuleListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all NormalizationRule objects.""" + + queryset = NormalizationRule.objects.all() + table = NormalizationRuleTable + filterset = NormalizationRuleFilterSet + filterset_form = NormalizationRuleFilterForm + template_name = "netbox_librenms_plugin/normalizationrule_list.html" + + +class NormalizationRuleCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + form = NormalizationRuleForm + + +@register_model_view(NormalizationRule, "bulk_import", path="import", detail=False) +class NormalizationRuleBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing NormalizationRule objects.""" + + queryset = NormalizationRule.objects.all() + model_form = NormalizationRuleImportForm + + +class NormalizationRuleView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + + +class NormalizationRuleEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + form = NormalizationRuleForm + + +class NormalizationRuleDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + + +class NormalizationRuleBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple NormalizationRule objects.""" + + queryset = NormalizationRule.objects.all() + table = NormalizationRuleTable + + +class NormalizationRuleChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() diff --git a/netbox_librenms_plugin/views/object_sync/__init__.py b/netbox_librenms_plugin/views/object_sync/__init__.py index e9893cf7d8..f025cb2a7b 100644 --- a/netbox_librenms_plugin/views/object_sync/__init__.py +++ b/netbox_librenms_plugin/views/object_sync/__init__.py @@ -5,6 +5,7 @@ DeviceInterfaceTableView, DeviceIPAddressTableView, DeviceLibreNMSSyncView, + DeviceModuleTableView, DeviceVLANTableView, SaveVlanGroupOverridesView, SingleInterfaceVerifyView, diff --git a/netbox_librenms_plugin/views/object_sync/devices.py b/netbox_librenms_plugin/views/object_sync/devices.py index 97584f8319..fe12c42751 100644 --- a/netbox_librenms_plugin/views/object_sync/devices.py +++ b/netbox_librenms_plugin/views/object_sync/devices.py @@ -17,6 +17,7 @@ LibreNMSInterfaceTable, VCInterfaceTable, ) +from netbox_librenms_plugin.tables.modules import LibreNMSModuleTable from netbox_librenms_plugin.utils import ( get_interface_name_field, get_missing_vlan_warning, @@ -29,6 +30,7 @@ from ..base.interfaces_view import BaseInterfaceTableView from ..base.ip_addresses_view import BaseIPAddressTableView from ..base.librenms_sync_view import BaseLibreNMSSyncView +from ..base.modules_view import BaseModuleTableView from ..base.vlan_table_view import BaseVLANTableView from ..mixins import CacheMixin, LibreNMSPermissionMixin @@ -63,6 +65,12 @@ def get_vlan_context(self, request, obj): vlan_table_view.request = request return vlan_table_view.get_vlan_context(request, obj) + def get_module_context(self, request, obj): + """Return module sync context for the device.""" + module_table_view = DeviceModuleTableView() + module_table_view.request = request + return module_table_view.get_context_data(request, obj) + class DeviceInterfaceTableView(BaseInterfaceTableView): """Interface synchronization table for Devices.""" @@ -79,13 +87,22 @@ def get_redirect_url(self, obj): def get_table(self, data, obj, interface_name_field, vlan_groups=None): """Return the appropriate interface table, selecting VC variant if needed.""" + server_key = self.librenms_api.server_key if hasattr(obj, "virtual_chassis") and obj.virtual_chassis: table = VCInterfaceTable( - data, device=obj, interface_name_field=interface_name_field, vlan_groups=vlan_groups + data, + device=obj, + interface_name_field=interface_name_field, + vlan_groups=vlan_groups, + server_key=server_key, ) else: table = LibreNMSInterfaceTable( - data, device=obj, interface_name_field=interface_name_field, vlan_groups=vlan_groups + data, + device=obj, + interface_name_field=interface_name_field, + vlan_groups=vlan_groups, + server_key=server_key, ) table.htmx_url = f"{self.request.path}?tab=interfaces" return table @@ -375,3 +392,15 @@ class DeviceVLANTableView(BaseVLANTableView): """VLAN synchronization table view for Devices.""" model = Device + + +class DeviceModuleTableView(BaseModuleTableView): + """Module/inventory synchronization view for Devices.""" + + model = Device + + def get_table(self, data, obj): + """Return the module sync table.""" + table = LibreNMSModuleTable(data, device=obj) + table.htmx_url = f"{self.request.path}?tab=modules" + return table diff --git a/netbox_librenms_plugin/views/object_sync/vms.py b/netbox_librenms_plugin/views/object_sync/vms.py index 51143d909d..bd6e052488 100644 --- a/netbox_librenms_plugin/views/object_sync/vms.py +++ b/netbox_librenms_plugin/views/object_sync/vms.py @@ -45,7 +45,9 @@ class VMInterfaceTableView(BaseInterfaceTableView): def get_table(self, data, obj, interface_name_field, vlan_groups=None): """Return a VM interface table for the given data.""" - return LibreNMSVMInterfaceTable(data, device=obj, vlan_groups=vlan_groups) + return LibreNMSVMInterfaceTable( + data, device=obj, vlan_groups=vlan_groups, server_key=self.librenms_api.server_key + ) def get_interfaces(self, obj): """Return all interfaces for the virtual machine.""" diff --git a/netbox_librenms_plugin/views/sync/cables.py b/netbox_librenms_plugin/views/sync/cables.py index 0e52f3b017..27dc5943c2 100644 --- a/netbox_librenms_plugin/views/sync/cables.py +++ b/netbox_librenms_plugin/views/sync/cables.py @@ -1,3 +1,5 @@ +import logging + from dcim.models import Cable, Device, Interface from django.contrib import messages from django.core.cache import cache @@ -9,6 +11,8 @@ from netbox_librenms_plugin.views.mixins import CacheMixin, LibreNMSPermissionMixin, NetBoxObjectPermissionMixin +logger = logging.getLogger(__name__) + class SyncCablesView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, CacheMixin, View): """Create NetBox cables using cached LibreNMS link data.""" @@ -42,7 +46,11 @@ def get_cached_links_data(self, request, obj): return cached_data.get("links", []) def create_cable(self, local_interface, remote_interface, request): - """Create a cable between local and remote interfaces.""" + """Create a cable between local and remote interfaces. + + Returns: + True on success, False on failure. + """ try: Cable.objects.create( a_terminations=[local_interface], @@ -81,13 +89,12 @@ def process_single_interface(self, interface, cached_links): link_data = next(link for link in cached_links if link["local_port"] == interface["interface"]) return self.handle_cable_creation(link_data, interface) except StopIteration: - return {"status": "invalid"} + return {"status": "invalid", "interface": interface.get("interface", "")} def verify_cable_creation_requirements(self, link_data): """Return True if all required NetBox IDs are present in link data.""" required_fields = [ "netbox_local_interface_id", - "netbox_remote_device_id", "netbox_remote_interface_id", ] @@ -113,13 +120,21 @@ def handle_cable_creation(self, link_data, interface): return {"status": "missing_remote", "interface": interface["interface"]} def process_interface_sync(self, selected_interfaces, cached_links): - """Process cable sync for all selected interfaces and return results.""" + """Process cable sync for all selected interfaces and return results. + + Each interface is processed in its own atomic block so individual + failures roll back only that cable without affecting others. + """ results = {"valid": [], "invalid": [], "duplicate": [], "missing_remote": []} - with transaction.atomic(): - for interface in selected_interfaces: - result = self.process_single_interface(interface, cached_links) + for interface in selected_interfaces: + try: + with transaction.atomic(): + result = self.process_single_interface(interface, cached_links) results[result["status"]].append(result.get("interface", "")) + except Exception: + logger.exception("Failed to sync cable for interface %s", interface.get("interface", "")) + results["invalid"].append(interface.get("interface", "")) return results diff --git a/netbox_librenms_plugin/views/sync/device_fields.py b/netbox_librenms_plugin/views/sync/device_fields.py index 952afdefa9..d621c7cec6 100644 --- a/netbox_librenms_plugin/views/sync/device_fields.py +++ b/netbox_librenms_plugin/views/sync/device_fields.py @@ -1,13 +1,63 @@ from dcim.models import Device, Manufacturer, Platform from django.contrib import messages from django.core.exceptions import ValidationError -from django.db import IntegrityError +from django.db import IntegrityError, transaction from django.shortcuts import get_object_or_404, redirect from django.views import View +import logging from netbox_librenms_plugin.utils import match_librenms_hardware_to_device_type from netbox_librenms_plugin.views.mixins import LibreNMSAPIMixin, LibreNMSPermissionMixin, NetBoxObjectPermissionMixin +logger = logging.getLogger(__name__) + + +class UpdateDeviceNameView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, LibreNMSAPIMixin, View): + """Update NetBox device name from LibreNMS sysName.""" + + required_object_permissions = { + "POST": [("change", Device)], + } + + def post(self, request, pk): + """Sync the device name from LibreNMS sysName.""" + if error := self.require_all_permissions("POST"): + return error + + device = get_object_or_404(Device, pk=pk) + self.librenms_id = self.librenms_api.get_librenms_id(device) + + if not self.librenms_id: + messages.error(request, "Device not found in LibreNMS") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + success, device_info = self.librenms_api.get_device_info(self.librenms_id) + + if not success or not device_info: + messages.error(request, "Failed to retrieve device info from LibreNMS") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + sys_name = device_info.get("sysName") + + if not sys_name: + messages.warning(request, "No sysName available in LibreNMS") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + old_name = device.name + device.name = sys_name + try: + device.full_clean() + device.save() + except (ValidationError, IntegrityError) as e: + device.name = old_name + error_msg = e.message_dict if hasattr(e, "message_dict") else str(e) + messages.error(request, f"Failed to update device name to '{sys_name}': {error_msg}") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + messages.success(request, f"Device name updated from '{old_name}' to '{sys_name}'") + + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + class UpdateDeviceSerialView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, LibreNMSAPIMixin, View): """Update NetBox device serial number from LibreNMS.""" @@ -231,28 +281,43 @@ def post(self, request, pk): pass try: - platform = Platform.objects.create( - name=platform_name, - manufacturer=manufacturer, + with transaction.atomic(): + platform = Platform.objects.create( + name=platform_name, + manufacturer=manufacturer, + ) + + device.platform = platform + device.full_clean() + device.save() + except IntegrityError as e: + error_str = str(e) + logger.error( + f"IntegrityError creating platform '{platform_name}' for device pk={pk}: {e}", + exc_info=True, + ) + if "platform" in error_str.lower() or "slug" in error_str.lower(): + messages.error( + request, + f"Platform '{platform_name}' could not be created (slug collision). Try a different name.", + ) + else: + messages.error( + request, + f"Failed to assign platform '{platform_name}'. Please contact an administrator.", + ) + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + except ValidationError as e: + logger.error( + f"ValidationError assigning platform '{platform_name}' to device pk={pk}: {e}", + exc_info=True, ) - except IntegrityError: messages.error( request, - f"Platform '{platform_name}' could not be created (slug collision). Try a different name.", + f"Failed to assign platform '{platform_name}'. Please contact an administrator.", ) return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) - old_platform = device.platform - device.platform = platform - try: - device.full_clean() - device.save() - except (ValidationError, IntegrityError) as e: - device.platform = old_platform - error_msg = e.message_dict if hasattr(e, "message_dict") else str(e) - messages.error(request, f"Failed to assign platform '{platform}': {error_msg}") - return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) - messages.success( request, f"Created platform '{platform}' and assigned to device", @@ -335,3 +400,73 @@ def post(self, request, pk): messages.info(request, "No serial assignments were made") return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + +class RemoveServerMappingView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, View): + """Remove a single server entry from the device's librenms_id custom field dict.""" + + required_object_permissions = { + "POST": [("change", Device)], + } + + def post(self, request, pk): + if error := self.require_all_permissions("POST"): + return error + + device = get_object_or_404(Device, pk=pk) + server_key = request.POST.get("server_key", "").strip() + + if not server_key: + messages.error(request, "No server_key provided.") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + cf_value = device.custom_field_data.get("librenms_id") + if not isinstance(cf_value, dict) or server_key not in cf_value: + messages.warning(request, f"No mapping found for server '{server_key}'.") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + # Refuse to remove mappings for servers that are still configured in the plugin. + # Only orphaned (unconfigured) mappings may be removed via this endpoint. + # Guard both multi-server mode (servers dict) and legacy single-server mode + # (top-level librenms_url in plugin config, which implicitly defines "default"). + from django.conf import settings as django_settings + + plugins_cfg = django_settings.PLUGINS_CONFIG.get("netbox_librenms_plugin", {}) + configured_servers = plugins_cfg.get("servers", {}) + legacy_url_configured = bool(plugins_cfg.get("librenms_url")) + if server_key in configured_servers or (legacy_url_configured and server_key == "default"): + messages.error( + request, + f"Cannot remove mapping for configured server '{server_key}'. " + "Remove the server from plugin configuration first, then retry.", + ) + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + with transaction.atomic(): + try: + device_locked = Device.objects.select_for_update().get(pk=pk) + except Device.DoesNotExist: + messages.error(request, "Device no longer exists.") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + cf = device_locked.custom_field_data.get("librenms_id", {}) + # Re-check after acquiring lock; mirror the pre-transaction protection logic + _is_protected = server_key in configured_servers or (legacy_url_configured and server_key == "default") + if isinstance(cf, dict) and server_key in cf and not _is_protected: + del cf[server_key] + device_locked.custom_field_data["librenms_id"] = cf if cf else None + try: + device_locked.full_clean() + device_locked.save() + except ValidationError as exc: + transaction.set_rollback(True) + messages.error(request, f"Validation error removing mapping: {exc}") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + except Exception as exc: + transaction.set_rollback(True) + messages.error(request, f"Error removing mapping for server '{server_key}': {exc}") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + messages.success(request, f"Removed LibreNMS mapping for server '{server_key}'.") + else: + messages.warning(request, f"Mapping for server '{server_key}' was already removed.") + + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) diff --git a/netbox_librenms_plugin/views/sync/devices.py b/netbox_librenms_plugin/views/sync/devices.py index da0f9af5b0..eb45b375ff 100644 --- a/netbox_librenms_plugin/views/sync/devices.py +++ b/netbox_librenms_plugin/views/sync/devices.py @@ -26,7 +26,7 @@ def get_object(self, object_id): try: return Device.objects.get(pk=object_id) except Device.DoesNotExist: - return VirtualMachine.objects.get(pk=object_id) + return get_object_or_404(VirtualMachine, pk=object_id) def post(self, request, object_id): """Add a device to LibreNMS using the submitted SNMP form.""" diff --git a/netbox_librenms_plugin/views/sync/interfaces.py b/netbox_librenms_plugin/views/sync/interfaces.py index 1100e0da1b..89bbc5ac92 100644 --- a/netbox_librenms_plugin/views/sync/interfaces.py +++ b/netbox_librenms_plugin/views/sync/interfaces.py @@ -9,7 +9,7 @@ from virtualization.models import VirtualMachine, VMInterface from netbox_librenms_plugin.models import InterfaceTypeMapping -from netbox_librenms_plugin.utils import convert_speed_to_kbps, get_interface_name_field +from netbox_librenms_plugin.utils import convert_speed_to_kbps, get_interface_name_field, set_librenms_device_id from netbox_librenms_plugin.views.mixins import ( CacheMixin, LibreNMSPermissionMixin, @@ -164,14 +164,8 @@ def sync_interface(self, obj, librenms_interface, exclude_columns, interface_nam ) # Sync VLANs if not excluded - vlan_synced = False if "vlans" not in exclude_columns: self._sync_interface_vlans(interface, librenms_interface, interface_name) - vlan_synced = True - - # Skip redundant save when _sync_interface_vlans already saved (via _update_interface_vlan_assignment) - if not vlan_synced: - interface.save() def get_netbox_interface_type(self, librenms_interface): """Return the NetBox interface type mapped from LibreNMS type and speed.""" @@ -235,7 +229,9 @@ def update_interface_attributes( setattr(interface, netbox_key, librenms_interface.get(librenms_key)) if "librenms_id" in interface.cf: - interface.custom_field_data["librenms_id"] = librenms_interface.get("port_id") + port_id = librenms_interface.get("port_id") + if port_id is not None: + set_librenms_device_id(interface, port_id, self.librenms_api.server_key) if "enabled" not in exclude_columns: admin_status = librenms_interface.get("ifAdminStatus") @@ -245,7 +241,7 @@ def update_interface_attributes( else (admin_status.lower() == "up" if isinstance(admin_status, str) else bool(admin_status)) ) - if "mac_address" not in exclude_columns: + if "mac_address" not in exclude_columns and is_device_interface: ifPhysAddress = librenms_interface.get("ifPhysAddress") self.handle_mac_address(interface, ifPhysAddress) diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 0000000000..62c68691ec --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,6 @@ +"""Conftest for e2e tests β€” no Django initialization needed.""" + +import os + +# Prevent pytest-django from trying to initialize Django +os.environ.pop("DJANGO_SETTINGS_MODULE", None) diff --git a/tests/e2e/test_module_install.py b/tests/e2e/test_module_install.py new file mode 100644 index 0000000000..cee05eb904 --- /dev/null +++ b/tests/e2e/test_module_install.py @@ -0,0 +1,330 @@ +"""End-to-end Playwright tests for LibreNMS plugin module sync workflow. + +These tests exercise the full import β†’ modules β†’ install flow against a +live NetBox + LibreNMS instance inside the devcontainer. + +Prerequisites: + - NetBox running at NETBOX_URL (default http://172.22.0.4:8000) + - LibreNMS server configured in plugin settings + - Device 15 (WS-C4900M) exists and is linked to LibreNMS + - Playwright installed: pip install playwright && playwright install chromium + +Run: + cd /home/mzieba/workspace/netbox-librenms-plugin + HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= \ + no_proxy=localhost,127.0.0.1,172.22.0.4 \ + python -m pytest tests/e2e/test_module_install.py -v -s +""" + +import os +import subprocess +import time + +import pytest + +NETBOX_URL = os.environ.get("NETBOX_URL", "http://172.22.0.4:8000") +NETBOX_USER = os.environ.get("NETBOX_USER", "admin") +NETBOX_PASS = os.environ.get("NETBOX_PASS", "admin") +CONTAINER_NAME = None + + +def _get_container(): + """Find the devcontainer name.""" + global CONTAINER_NAME + if CONTAINER_NAME: + return CONTAINER_NAME + result = subprocess.run( + ["docker", "ps", "--format", "{{.Names}}"], + capture_output=True, + text=True, + ) + for name in result.stdout.strip().split("\n"): + if "devcontainer-devcontainer" in name: + CONTAINER_NAME = name + return name + pytest.skip("No devcontainer found") + + +def _netbox_shell(code): + """Run Python code in NetBox's Django shell.""" + import shlex + + container = _get_container() + escaped = shlex.quote(code) + result = subprocess.run( + [ + "docker", + "exec", + container, + "bash", + "-c", + f"cd /opt/netbox/netbox && python3 manage.py shell -c {escaped}", + ], + capture_output=True, + text=True, + env={"PATH": "/usr/bin:/bin", "HOME": "/root"}, + ) + # Filter out config loading lines + lines = [line for line in result.stdout.strip().split("\n") if not line.startswith(("🧬", "156 objects"))] + return "\n".join(lines).strip() + + +def _delete_device_modules(device_id): + """Remove all modules from a device.""" + _netbox_shell( + f"from dcim.models import Module; " + f"deleted = Module.objects.filter(device_id={device_id}).delete(); " + f"print(f'Deleted {{deleted}}')" + ) + + +def _get_interfaces(device_id): + """Get interface names for a device.""" + output = _netbox_shell( + f"from dcim.models import Interface; " + f'[print(f\'{{i.name}}|{{i.module.module_type.model if i.module else "-"}}|' + f'{{i.module.module_bay.name if i.module else "-"}}\')' + f" for i in Interface.objects.filter(device_id={device_id}).order_by('name')]" + ) + results = [] + for line in output.split("\n"): + if "|" in line: + name, mod_type, bay = line.split("|") + results.append({"name": name, "module_type": mod_type, "bay": bay}) + return results + + +@pytest.fixture(scope="module") +def browser(): + """Launch browser for the test module.""" + from playwright.sync_api import sync_playwright + + pw = sync_playwright().start() + b = pw.chromium.launch(headless=True) + yield b + b.close() + pw.stop() + + +@pytest.fixture +def page(browser): + """Create a new page and log in to NetBox.""" + ctx = browser.new_context(ignore_https_errors=True) + pg = ctx.new_page() + + pg.goto(f"{NETBOX_URL}/login/", timeout=10000) + pg.fill("#id_username", NETBOX_USER) + pg.fill("#id_password", NETBOX_PASS) + pg.click("button[type=submit]") + pg.wait_for_load_state("networkidle") + yield pg + ctx.close() + + +class TestModuleInstallWorkflow: + """Test the full module sync and install workflow on device 15 (WS-C4900M).""" + + DEVICE_ID = 15 + + def _goto_modules_tab(self, page): + """Navigate to the modules sync tab and refresh data.""" + page.goto(f"{NETBOX_URL}/dcim/devices/{self.DEVICE_ID}/librenms-sync/?tab=modules") + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Click Refresh Modules + btn = page.query_selector('button:has-text("Refresh Modules")') + assert btn is not None, "Refresh Modules button not found" + btn.click() + time.sleep(8) + + def _get_table_rows(self, page): + """Parse the module sync table into dicts.""" + pane = page.query_selector("#modules") + assert pane is not None, "Modules pane not found" + + rows = [] + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if len(cells) >= 8: + rows.append( + { + "name": cells[0].inner_text().strip(), + "model": cells[1].inner_text().strip(), + "serial": cells[2].inner_text().strip(), + "bay": cells[5].inner_text().strip(), + "type": cells[6].inner_text().strip(), + "status": cells[7].inner_text().strip(), + } + ) + return rows + + def test_clean_state_shows_install_buttons(self, page): + """After deleting all modules, table shows Install buttons.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + rows = self._get_table_rows(page) + assert len(rows) > 0, "No rows in module sync table" + + # Top-level items with matched bays should show Matched status + supervisor = [r for r in rows if "Supervisor(slot 1)" in r["name"]] + assert len(supervisor) == 1, f"Expected 1 Supervisor row, got {len(supervisor)}" + assert supervisor[0]["status"] == "Matched", f"Expected Matched, got {supervisor[0]['status']}" + + def test_single_install(self, page): + """Installing a single top-level module works.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + # Install FanTray 1 + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "FanTray 1" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install"):not(:has-text("Branch"))') + if btn: + btn.click() + time.sleep(5) + break + + # Verify via DB + output = _netbox_shell( + f"from dcim.models import Module; " + f"m = Module.objects.filter(device_id={self.DEVICE_ID}, module_bay__name='Fan Tray 1').first(); " + f"print(m.module_type.model if m else 'NONE')" + ) + assert "WS-X4992" in output, f"FanTray not installed: {output}" + + def test_branch_install_supervisor(self, page): + """Branch install creates supervisor + X2 transceivers with correct names.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + # Click Install Branch on Supervisor(slot 1) + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Supervisor(slot 1)" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install Branch")') + assert btn is not None, "Install Branch button not found for Supervisor" + btn.click() + break + + # Wait for branch install to complete (creates many modules + signals) + time.sleep(20) + page.wait_for_load_state("networkidle") + time.sleep(5) + + # Verify interfaces have correct names (not bare position numbers) + interfaces = _get_interfaces(self.DEVICE_ID) + x2_interfaces = [i for i in interfaces if i["module_type"] in ("X2-10GB-LR", "X2-10GB-SR")] + + assert len(x2_interfaces) > 0, "No X2 transceiver interfaces created" + + for iface in x2_interfaces: + assert iface["name"].startswith("TenGigabitEthernet"), ( + f"Interface '{iface['name']}' in {iface['bay']} " + f"should start with 'TenGigabitEthernet' (INR rule not applied?)" + ) + + def test_branch_install_no_duplicate_errors(self, page): + """Branch install handles already-occupied bays gracefully.""" + # Don't delete modules β€” some should already be installed + self._goto_modules_tab(page) + + # Click Install Branch on Supervisor(slot 1) again + pane = page.query_selector("#modules") + branch_btn = None + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Supervisor(slot 1)" in cells[0].inner_text(): + branch_btn = tr.query_selector('button:has-text("Install Branch")') + break + + if branch_btn: + branch_btn.click() + time.sleep(10) + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Check for error messages β€” should only have skips, no failures + body_text = page.query_selector("body").inner_text() + assert "Branch install failed" not in body_text, ( + "Branch install crashed instead of handling errors gracefully" + ) + + def test_child_bays_hidden_when_parent_not_installed(self, page): + """Children show 'No Bay' when parent module is not installed.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + rows = self._get_table_rows(page) + + # Children of Supervisor(slot 1) should show "No matching bay" + # since Supervisor isn't installed, its child bays don't exist yet + children = [r for r in rows if r["name"].startswith("└─") and "TenGigabitEthernet1/" in r["name"]] + for child in children: + assert "No matching bay" in child["bay"], ( + f"Child '{child['name']}' should show 'No matching bay' when parent not installed, got '{child['bay']}'" + ) + + def test_full_workflow(self, page): + """Full workflow: clean β†’ install individuals β†’ branch install β†’ verify.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + # Step 1: Install PSUs and FanTray individually + for label in ["FanTray 1", "Power Supply 1", "Power Supply 2"]: + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and label in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install"):not(:has-text("Branch"))') + if btn: + btn.click() + time.sleep(4) + break + + # Step 2: Branch install Supervisor + transceivers + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Supervisor(slot 1)" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install Branch")') + if btn: + btn.click() + time.sleep(10) + break + + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Step 3: Branch install Linecard + self._goto_modules_tab(page) + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Linecard(slot 3)" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install Branch")') + if btn: + btn.click() + time.sleep(10) + break + + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Verify: all installable modules should be installed + self._goto_modules_tab(page) + rows = self._get_table_rows(page) + + matched_but_not_installed = [r for r in rows if r["status"] == "Matched" and not r["name"].startswith("└─")] + assert len(matched_but_not_installed) == 0, ( + f"Top-level items still 'Matched' after full workflow: {[r['name'] for r in matched_but_not_installed]}" + ) + + # Verify interface naming + interfaces = _get_interfaces(self.DEVICE_ID) + for iface in interfaces: + assert iface["name"] != "1", "Interface with bare name '1' found β€” INR rule not applied" diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000000..4f6c48c6f1 --- /dev/null +++ b/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.12.0" + +[[package]] +name = "netbox-librenms-plugin" +version = "0.4.2" +source = { editable = "." }