From 784072f649ba4da2463ac01ae13ded72ee50e384 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 26 Feb 2026 22:44:18 +0100 Subject: [PATCH 01/39] refactor: split import_utils.py into a package Break the monolithic import_utils.py into focused modules: - permissions.py: user permission checks - cache.py: cache key generation and management - filters.py: device filtering and retrieval from LibreNMS - virtual_chassis.py: VC detection, creation, member management - device_operations.py: device validation, import, and fetch - vm_operations.py: VM creation and bulk import - bulk_import.py: bulk device import orchestration and filter processing The __init__.py re-exports all public names, so existing callers (views, jobs, tests) continue working without import changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- netbox_librenms_plugin/import_utils.py | 2484 ----------------- .../import_utils/__init__.py | 52 + .../import_utils/bulk_import.py | 563 ++++ netbox_librenms_plugin/import_utils/cache.py | 158 ++ .../import_utils/device_operations.py | 859 ++++++ .../import_utils/filters.py | 253 ++ .../import_utils/permissions.py | 48 + .../import_utils/virtual_chassis.py | 440 +++ .../import_utils/vm_operations.py | 216 ++ 9 files changed, 2589 insertions(+), 2484 deletions(-) delete mode 100644 netbox_librenms_plugin/import_utils.py create mode 100644 netbox_librenms_plugin/import_utils/__init__.py create mode 100644 netbox_librenms_plugin/import_utils/bulk_import.py create mode 100644 netbox_librenms_plugin/import_utils/cache.py create mode 100644 netbox_librenms_plugin/import_utils/device_operations.py create mode 100644 netbox_librenms_plugin/import_utils/filters.py create mode 100644 netbox_librenms_plugin/import_utils/permissions.py create mode 100644 netbox_librenms_plugin/import_utils/virtual_chassis.py create mode 100644 netbox_librenms_plugin/import_utils/vm_operations.py diff --git a/netbox_librenms_plugin/import_utils.py b/netbox_librenms_plugin/import_utils.py deleted file mode 100644 index feac6e3e99..0000000000 --- a/netbox_librenms_plugin/import_utils.py +++ /dev/null @@ -1,2484 +0,0 @@ -""" -Utilities for importing devices from LibreNMS to NetBox. - -This module provides functions for: -- Validating LibreNMS devices for import -- Retrieving filtered LibreNMS devices -- Importing single and multiple devices -- Smart matching of NetBox objects -- Permission checking for import operations -""" - -import logging -from typing import List - -from core.choices import JobStatusChoices -from dcim.models import Device, DeviceRole, DeviceType, Rack, Site, VirtualChassis -from django.core.cache import cache -from django.core.exceptions import PermissionDenied -from django.db import transaction -from django.utils import timezone -from virtualization.models import Cluster - -from .librenms_api import LibreNMSAPI -from .utils import ( - find_matching_platform, - find_matching_site, - match_librenms_hardware_to_device_type, -) - -logger = logging.getLogger(__name__) - - -# ============================================================================= -# Permission Check Helpers -# ============================================================================= - - -def check_user_permissions(user, permissions): - """ - Check if user has all required permissions. - - Args: - user: The user object to check permissions for - permissions: List of permission strings (e.g., ['dcim.add_device', 'dcim.add_interface']) - - Returns: - tuple: (has_all_permissions: bool, missing_permissions: list[str]) - - Raises: - PermissionDenied: If user is None (no user context available) - """ - if user is None: - raise PermissionDenied("No user context available for permission check") - - missing = [perm for perm in permissions if not user.has_perm(perm)] - return (len(missing) == 0, missing) - - -def require_permissions(user, permissions, action_description="perform this action"): - """ - Require user has all permissions, raising PermissionDenied if not. - - Args: - user: The user object to check permissions for - permissions: List of permission strings - action_description: Human-readable description for error message - - Raises: - PermissionDenied: If user lacks any required permission - """ - has_perms, missing = check_user_permissions(user, permissions) - if not has_perms: - missing_str = ", ".join(missing) - raise PermissionDenied( - f"You do not have permission to {action_description}. Missing permissions: {missing_str}" - ) - - -def get_cache_metadata_key(server_key: str, filters: dict, vc_enabled: bool) -> str: - """ - Generate a consistent cache metadata key from filter parameters. - - Args: - server_key: LibreNMS server identifier - filters: Filter dictionary - vc_enabled: Whether VC detection is enabled - - Returns: - str: Consistent cache key for metadata - """ - # Sort filter items to ensure consistent key generation - filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v) - return f"librenms_filter_cache_metadata_{server_key}_{filter_parts}_{vc_enabled}" - - -def get_active_cached_searches(server_key: str) -> list[dict]: - """ - Retrieve all active cached searches for a server and enrich with display-friendly values. - - Enriches raw filter IDs with human-readable names by looking up location names - from cached choices and converting type codes to display names. - - Args: - server_key: LibreNMS server identifier - - Returns: - List of dicts containing cache metadata with enriched display_filters - """ - from datetime import datetime, timezone - - cache_index_key = f"librenms_cache_index_{server_key}" - cache_index = cache.get(cache_index_key, []) - - active_searches = [] - valid_cache_keys = [] - - # Get location and type choices for enriching display - location_choices = {} - type_choices = { - "": "All Types", - "network": "Network", - "server": "Server", - "storage": "Storage", - "wireless": "Wireless", - "firewall": "Firewall", - "power": "Power", - "appliance": "Appliance", - "printer": "Printer", - "loadbalancer": "Load Balancer", - "other": "Other", - } - - # Get cached location choices for enrichment - location_cache_key = "librenms_locations_choices" - cached_locations = cache.get(location_cache_key) - if cached_locations: - location_choices = dict(cached_locations) - - for cache_key in cache_index: - metadata = cache.get(cache_key) - if metadata: - # Cache still exists, calculate time remaining - cached_at = datetime.fromisoformat(metadata.get("cached_at")) - cache_timeout = metadata.get("cache_timeout", 300) - now = datetime.now(timezone.utc) - age_seconds = (now - cached_at).total_seconds() - remaining_seconds = max(0, cache_timeout - age_seconds) - - if remaining_seconds > 0: - # Add remaining time and cache key - metadata["remaining_seconds"] = int(remaining_seconds) - metadata["cache_key"] = cache_key - - # Enrich filters with human-readable display values - if "filters" in metadata: - display_filters = metadata["filters"].copy() - # Convert location ID to location name - if "location" in display_filters and display_filters["location"] in location_choices: - display_filters["location"] = location_choices[display_filters["location"]] - # Convert type code to display name - if "type" in display_filters and display_filters["type"] in type_choices: - display_filters["type"] = type_choices[display_filters["type"]] - metadata["display_filters"] = display_filters - else: - # Fallback if filters key missing - metadata["display_filters"] = {} - - active_searches.append(metadata) - valid_cache_keys.append(cache_key) - - # Clean up index if any keys have expired - if len(valid_cache_keys) < len(cache_index): - cache.set(cache_index_key, valid_cache_keys, timeout=3600) - - # Sort by most recent first - active_searches.sort(key=lambda x: x.get("cached_at", ""), reverse=True) - - return active_searches - - -def get_validated_device_cache_key(server_key: str, filters: dict, device_id: int | str, vc_enabled: bool) -> str: - """ - Generate a consistent cache key for validated device data. - - This ensures both synchronous and background job processing use the same - cache keys, avoiding duplicate validation work and cache entries. - - Args: - server_key: LibreNMS server key - filters: Filter dict with location, type, os, hostname, sysname, hardware keys - device_id: LibreNMS device ID - vc_enabled: Whether virtual chassis detection was enabled - - Returns: - str: Cache key for the validated device - - Example: - >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) - >>> key - 'validated_device_default_-1234567890_123_vc' - """ - # Sort filters for consistent hashing - filter_hash = hash(str(sorted(filters.items()))) - vc_part = "vc" if vc_enabled else "novc" - return f"validated_device_{server_key}_{filter_hash}_{device_id}_{vc_part}" - - -def get_import_device_cache_key(device_id: int | str, server_key: str = "default") -> str: - """ - Generate cache key for raw LibreNMS device data. - - This key is used to cache raw device data (without validation metadata) - to avoid redundant API calls when users interact with dropdowns during - the import workflow. - - Args: - device_id: LibreNMS device ID - server_key: LibreNMS server identifier for multi-server setups - - Returns: - str: Cache key for the device data - - Example: - >>> get_import_device_cache_key(123, "production") - 'import_device_data_production_123' - """ - return f"import_device_data_{server_key}_{device_id}" - - -def _determine_device_name( - libre_device: dict, - use_sysname: bool = True, - strip_domain: bool = False, - device_id: int | str = None, -) -> str: - """ - Determine the device/VM name from LibreNMS data. - - Centralized logic for building device names with consistent handling of: - - sysName vs hostname preference - - Domain stripping (avoiding IP addresses) - - Fallback to device_id when name is missing - - Args: - libre_device: Device data from LibreNMS - use_sysname: If True, prefer sysName; if False, use hostname - strip_domain: If True, strip domain suffix (e.g., '.example.com') - device_id: LibreNMS device ID for fallback name generation - - Returns: - str: The determined device name - - Example: - >>> _determine_device_name({'sysName': 'router.example.com', 'hostname': 'router'}, - ... use_sysname=True, strip_domain=True) - 'router' - """ - # Determine base name based on use_sysname preference - if use_sysname: - name = libre_device.get("sysName") or libre_device.get("hostname") - else: - name = libre_device.get("hostname") or libre_device.get("sysName") - - # Fallback to device_id if no name found - if not name: - if device_id is not None: - name = f"device-{device_id}" - else: - name = libre_device.get("device_id", "unknown") - name = f"device-{name}" - - # Strip domain if requested (but not for IP addresses) - if strip_domain and name and "." in name: - try: - from ipaddress import ip_address - - ip_address(name) - # It's a valid IP address, don't strip - except ValueError: - # Not an IP, safe to strip domain - name = name.split(".")[0] - - return name - - -def empty_virtual_chassis_data() -> dict: - """Public helper for callers that need a blank VC payload.""" - - return { - "is_stack": False, - "member_count": 0, - "members": [], - "detection_error": None, - } - - -def _clone_virtual_chassis_data(data: dict | None) -> dict: - """Return a defensive copy of cached VC data to avoid shared references.""" - - if not data: - return empty_virtual_chassis_data() - - members = [] - for idx, member in enumerate(data.get("members", [])): - member_copy = member.copy() - raw_position = member_copy.get("position", idx) - try: - member_copy["position"] = int(raw_position) - except (TypeError, ValueError): - member_copy["position"] = idx - members.append(member_copy) - - member_count = data.get("member_count") or len(members) - - return { - "is_stack": bool(data.get("is_stack")), - "member_count": member_count, - "members": members, - "detection_error": data.get("detection_error"), - } - - -_VC_CACHE_VERSION = "v1" - - -def _vc_cache_key(api: LibreNMSAPI, device_id: int | str) -> str: - server_key = getattr(api, "server_key", "default") - return f"librenms_vc_detection_{_VC_CACHE_VERSION}_{server_key}_{device_id}" - - -def get_virtual_chassis_data(api: LibreNMSAPI, device_id: int | str, *, force_refresh: bool = False) -> dict: - """Fetch (and cache) virtual chassis data for a LibreNMS device.""" - - if not api or device_id is None: - return empty_virtual_chassis_data() - - cache_key = _vc_cache_key(api, device_id) - if not force_refresh: - cached = cache.get(cache_key) - if cached is not None: - return _clone_virtual_chassis_data(cached) - - detection_data = detect_virtual_chassis_from_inventory(api, device_id) - if detection_data and "detection_error" not in detection_data: - detection_data["detection_error"] = None - - cache_value = _clone_virtual_chassis_data(detection_data) if detection_data else empty_virtual_chassis_data() - - cache_timeout = getattr(api, "cache_timeout", 300) or 300 - cache.set(cache_key, cache_value, timeout=cache_timeout) - return _clone_virtual_chassis_data(cache_value) - - -def prefetch_vc_data_for_devices(api: LibreNMSAPI, device_ids: List[int], *, force_refresh: bool = False) -> None: - """ - Pre-warm the virtual chassis cache for multiple devices. - - This eliminates the 0.5-1s delay when rendering the import table - by proactively fetching VC data before validation. - - Args: - api: LibreNMSAPI instance - device_ids: List of LibreNMS device IDs to prefetch VC data for - force_refresh: When True, bypass cache and fetch fresh data - - Example: - >>> # Before rendering import table - >>> prefetch_vc_data_for_devices(api, [123, 124, 125]) - >>> # Now all validate_device_for_import() calls hit cache instantly - """ - if not api or not device_ids: - return - - logger.debug(f"Pre-warming VC cache for {len(device_ids)} devices") - - for idx, device_id in enumerate(device_ids): - # This populates the cache if empty, or skips if already cached - try: - get_virtual_chassis_data(api, device_id, force_refresh=force_refresh) - except (BrokenPipeError, ConnectionError, IOError, OSError) as e: - logger.warning(f"Connection error during VC prefetch at device {idx}: {e}") - # Stop processing if connection is broken - return - except Exception as e: - # Log but continue for other errors - logger.warning(f"Error prefetching VC data for device {device_id}: {e}") - - logger.debug(f"VC cache warming complete for {len(device_ids)} devices") - - -def get_device_count_for_filters( - api: LibreNMSAPI, - filters: dict, - clear_cache: bool = False, - show_disabled: bool = True, -) -> int: - """ - Get count of LibreNMS devices matching filters. - - This is a lightweight function to determine device count for background job - decision making. Uses the same caching as get_librenms_devices_for_import(). - - Args: - api: LibreNMS API client instance - filters: Filter dict with location, type, os, hostname, sysname keys - clear_cache: Whether to force cache refresh - show_disabled: Whether to include disabled devices - - Returns: - int: Count of devices matching filters - """ - devices = get_librenms_devices_for_import(api, filters=filters, force_refresh=clear_cache) - - # Filter out disabled devices if requested - if not show_disabled: - devices = [d for d in devices if d.get("status") == 1] - - return len(devices) - - -def get_librenms_devices_for_import( - api: LibreNMSAPI = None, - filters: dict = None, - server_key: str = None, - *, - force_refresh: bool = False, - return_cache_status: bool = False, -) -> List[dict] | tuple[List[dict], bool]: - """ - Retrieve LibreNMS devices based on filters. - - Args: - api: LibreNMSAPI instance (if not provided, creates one with server_key) - filters: Dict containing filter parameters: - - location: LibreNMS location/site filter - - type: Device type filter - - os: Operating system filter - - hostname: Hostname filter (partial match) - - sysname: System name filter (partial match) - - status: Device status filter (1=up, 0=down) - - disabled: Include disabled devices (0=active only, 1=all) - server_key: Key for specific server configuration (used if api not provided) - force_refresh: When True, bypass the cache and fetch fresh data - return_cache_status: When True, returns (devices, from_cache) tuple - - Returns: - List of device dictionaries from LibreNMS, or tuple of (devices, from_cache) - if return_cache_status is True. from_cache=True means data was loaded from - existing cache; from_cache=False means data was just fetched from LibreNMS. - """ - try: - # Use provided API instance or create a new one - if api is None: - api = LibreNMSAPI(server_key=server_key) - - # Build LibreNMS API filters using the type/query format - # LibreNMS API v0 expects ?type=X&query=Y format, not direct parameters - # NOTE: API only supports ONE type/query pair, so we'll use the most - # specific filter for the API and apply others client-side - api_filters = {} - client_filters = {} # Filters to apply after fetching from API - - if filters: - # Check for status filter first - it has special handling - if filters.get("status") is not None: - # Status filter uses special types that don't need query param - if filters["status"] == 1: - api_filters["type"] = "up" - elif filters["status"] == 0: - api_filters["type"] = "down" - - # Save ALL other filters for client-side filtering when status is used - if filters.get("location"): - client_filters["location"] = filters["location"] - if filters.get("type"): - client_filters["type"] = filters["type"] - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - else: - # Priority order for type/query filters: location > type > os > hostname > sysname - # Note: When sysname is combined with other filters, it's applied client-side for partial matching - # When sysname is alone, it uses API exact match (type=sysName) - # Note: hardware is always applied client-side for partial matching - # Use first available for API, save others for client-side filtering - if filters.get("location"): - api_filters["type"] = "location_id" - api_filters["query"] = filters["location"] - # Save remaining filters for client-side - if filters.get("type"): - client_filters["type"] = filters["type"] - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("type"): - api_filters["type"] = "type" - api_filters["query"] = filters["type"] - # Save remaining filters for client-side - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("os"): - api_filters["type"] = "os" - api_filters["query"] = filters["os"] - # Save remaining filters for client-side - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("hostname"): - api_filters["type"] = "hostname" - api_filters["query"] = filters["hostname"] - # Save sysname and hardware for client-side - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("sysname"): - # sysname-only filter: Use API exact match (type=sysName&query=) - # This is safe - returns empty if no exact match found - api_filters["type"] = "sysName" - api_filters["query"] = filters["sysname"] - # Save hardware for client-side - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("hardware"): - # hardware-only filter: apply client-side for partial matching - client_filters["hardware"] = filters["hardware"] - - # Note: disabled filter isn't directly supported by LibreNMS API - # We'll filter client-side if needed - - # Use caching to avoid repeated API calls - # Include both API and client filters in cache key - cache_key = f"librenms_devices_import_{server_key}_{hash(str(api_filters))}_{hash(str(client_filters))}" - from_cache = False - - if force_refresh: - cache.delete(cache_key) - else: - cached_result = cache.get(cache_key) - if cached_result is not None: - # No need to deepcopy - cached data isn't mutated - devices = cached_result - from_cache = True - if return_cache_status: - return devices, from_cache - return devices - - success, devices = api.list_devices(api_filters if api_filters else None) - - if not success: - logger.error(f"Failed to retrieve devices from LibreNMS: {devices}") - if return_cache_status: - return [], False - return [] - - # Apply client-side filters if any - if client_filters: - devices = _apply_client_filters(devices, client_filters) - - # Cache using configured timeout (default 300s) - # No need to deepcopy - Django's cache backend handles serialization - cache.set(cache_key, devices, timeout=api.cache_timeout) - - if return_cache_status: - return devices, from_cache - return devices - - except Exception: - logger.exception("Error retrieving LibreNMS devices for import") - if return_cache_status: - return [], False - return [] - - -def _apply_client_filters(devices: List[dict], filters: dict) -> List[dict]: - """ - Apply client-side filters to device list. - - Args: - devices: List of device dicts from LibreNMS - filters: Dict of filters to apply (location, type, os, hostname, sysname) - - Returns: - Filtered list of devices - """ - filtered = devices - - if filters.get("location"): - location_id = str(filters["location"]) - filtered = [d for d in filtered if str(d.get("location_id", "")) == location_id] - - if filters.get("type"): - device_type = filters["type"].lower() - filtered = [d for d in filtered if d.get("type", "").lower() == device_type] - - if filters.get("os"): - os_filter = filters["os"].lower() - filtered = [d for d in filtered if os_filter in d.get("os", "").lower()] - - if filters.get("hostname"): - hostname_filter = filters["hostname"].lower() - filtered = [d for d in filtered if hostname_filter in d.get("hostname", "").lower()] - - if filters.get("sysname"): - sysname_filter = filters["sysname"].lower() - filtered = [d for d in filtered if sysname_filter in d.get("sysName", "").lower()] - - if filters.get("hardware"): - hardware_filter = filters["hardware"].lower() - filtered = [d for d in filtered if hardware_filter in (d.get("hardware") or "").lower()] - - return filtered - - -def validate_device_for_import( - libre_device: dict, - import_as_vm: bool = False, - api: "LibreNMSAPI" = None, - *, - include_vc_detection: bool = True, - force_vc_refresh: bool = False, -) -> dict: - """ - Validate if a LibreNMS device can be imported to NetBox. - - Performs comprehensive validation: - - Checks if device already exists in NetBox - - Validates required prerequisites (Site, DeviceType, DeviceRole for devices) - OR (Cluster for VMs) - - Provides smart matching for missing objects - - Detects virtual chassis/stack configuration (if API provided) - - Returns detailed validation status - - Args: - libre_device: Device data from LibreNMS - import_as_vm: If True, validate for VM import instead of device import - api: Optional LibreNMSAPI instance for virtual chassis detection - include_vc_detection: Skip VC detection when False to speed up bulk operations - force_vc_refresh: When True, bypass cached VC data and re-query LibreNMS - - Returns: - dict: Validation result with structure: - { - 'is_ready': bool, # Can import without user intervention - 'can_import': bool, # Can import (possibly after configuration) - 'import_as_vm': bool, # Whether importing as VM - 'existing_device': Device or VirtualMachine or None, - 'issues': List[str], # Blocking issues - 'warnings': List[str], # Non-blocking warnings - 'site': { # Only for devices - 'found': bool, - 'site': Site or None, - 'match_type': str, # 'exact' or None - 'suggestions': List[Site] # Alternative suggestions - }, - 'device_type': { # Only for devices - 'found': bool, - 'device_type': DeviceType or None, - 'match_type': str, # 'exact' or None - 'suggestions': List[dict] # Device types for user selection - }, - 'device_role': { # Only for devices - 'found': bool, # Always False - requires manual selection - 'role': DeviceRole or None, - 'available_roles': List[DeviceRole] # All roles for user selection - }, - 'cluster': { # Only for VMs - 'found': bool, # Always False - requires manual selection - 'cluster': Cluster or None, - 'available_clusters': List[Cluster] # All clusters for user selection - }, - 'platform': { - 'found': bool, - 'platform': Platform or None, - 'match_type': str # 'exact' or None - } - } - - Example: - >>> validation = validate_device_for_import(libre_device) - >>> if validation['is_ready']: - ... import_single_device(libre_device['device_id']) - """ - result = { - "is_ready": False, - "can_import": False, - "import_as_vm": import_as_vm, - "existing_device": None, - "existing_match_type": None, # Track how existing device was matched - "serial_action": None, # None, "link", "conflict", "update_serial", "hostname_differs" - "serial_confirmed": False, # True when librenms_id match and serial matches - "serial_duplicate": False, # True when incoming serial is already on a different device - "name_matches": False, # True when existing device name matches LibreNMS sysName - "name_sync_available": False, # True when existing device name differs from sysName - "suggested_name": None, # sysName to suggest when name_sync_available is True - "device_type_mismatch": False, # True when existing device's type differs from LibreNMS - "issues": [], - "warnings": [], - "virtual_chassis": empty_virtual_chassis_data(), - "site": { - "found": False, - "site": None, - "match_type": None, - "suggestions": [], - }, - "device_type": { - "found": False, - "device_type": None, - "match_type": None, - "suggestions": [], - }, - "device_role": { - "found": False, - "role": None, - "available_roles": [], - }, - "cluster": { - "found": False, - "cluster": None, - "available_clusters": [], - }, - "platform": {"found": False, "platform": None, "match_type": None}, - "rack": { - "found": False, - "rack": None, - "available_racks": [], - }, - } - - try: - # 1. Check if device/VM already exists in NetBox - # Always check both Devices AND VMs to properly detect existing objects - librenms_id = libre_device.get("device_id") - hostname = libre_device.get("hostname", "") - logger.debug( - f"Checking for existing device/VM: " - f"librenms_id={librenms_id} (type={type(librenms_id).__name__}), " - f"hostname={hostname}" - ) - - from virtualization.models import VirtualMachine - - # Check for existing VM first (by librenms_id custom field) - # Always query with int to match custom field type - try: - existing_vm = VirtualMachine.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() - except (ValueError, TypeError): - # librenms_id is not convertible to int; no match will be found - existing_vm = None - - if existing_vm: - logger.info(f"Found existing VM: {existing_vm.name} (matched by librenms_id={librenms_id})") - result["existing_device"] = existing_vm - result["existing_match_type"] = "librenms_id" - result["import_as_vm"] = True # Force VM mode since VM exists - result["can_import"] = False - - # Check if name matches sysName - # Note: name_sync_available/suggested_name are intentionally not set for VMs - # because UpdateDeviceNameView only supports Device objects; VM name-sync - # would require a separate implementation. - sys_name = libre_device.get("sysName") or "" - if sys_name and existing_vm.name == sys_name: - result["name_matches"] = True - - # Check for existing Device (by librenms_id custom field) - # Always query with int to match custom field type - if not result["existing_device"]: - try: - existing_device = Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() - except (ValueError, TypeError): - # librenms_id is not convertible to int; no match will be found - existing_device = None - - if existing_device: - logger.info(f"Found existing device: {existing_device.name} (matched by librenms_id={librenms_id})") - result["existing_device"] = existing_device - result["existing_match_type"] = "librenms_id" - result["can_import"] = False - - # Check if name matches sysName - sys_name = libre_device.get("sysName") or "" - if sys_name and existing_device.name == sys_name: - result["name_matches"] = True - elif sys_name and existing_device.name != sys_name: - result["name_sync_available"] = True - result["suggested_name"] = sys_name - - # Check for serial drift on the linked device - incoming_serial = libre_device.get("serial") or "" - if incoming_serial and incoming_serial != "-": - if existing_device.serial and existing_device.serial == incoming_serial: - result["serial_confirmed"] = True - elif existing_device.serial and existing_device.serial != incoming_serial: - serial_conflict = ( - Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() - ) - if serial_conflict: - result["serial_action"] = "conflict" - result["serial_duplicate"] = True - result["warnings"].append( - f"Serial conflict: incoming serial '{incoming_serial}' is already assigned to " - f"device '{serial_conflict.name}' (ID: {serial_conflict.pk}) in NetBox. " - f"Investigate which device should own this serial before updating." - ) - else: - result["serial_action"] = "update_serial" - result["warnings"].append( - f"Serial number differs (NetBox: '{existing_device.serial}', " - f"LibreNMS: '{incoming_serial}'). Hardware may have been replaced." - ) - - # Only check hostname/serial/IP if not already matched by librenms_id - if not result["existing_device"]: - # Check by hostname/name - Check both VMs and Devices for conflicts - existing_vm = VirtualMachine.objects.filter(name__iexact=hostname).first() - existing_device = Device.objects.filter(name__iexact=hostname).first() - - # If BOTH exist with same hostname, it's ambiguous - don't match either - if existing_vm and existing_device: - logger.warning( - f"Hostname conflict: Both VM '{existing_vm.name}' and Device " - f"'{existing_device.name}' exist with hostname '{hostname}'" - ) - result["warnings"].append( - f"Both a VM and Device exist with hostname '{hostname}' in NetBox. " - f"Cannot determine which to match. Please set the librenms_id custom field on the correct object." - ) - # Don't set existing_device, don't block import - let user proceed as new - # This allows them to import and then resolve the conflict manually - elif existing_vm: - logger.info(f"Found existing VM by hostname: {existing_vm.name}") - result["existing_device"] = existing_vm - result["existing_match_type"] = "hostname" - result["import_as_vm"] = True # Force VM mode since VM exists - result["warnings"].append( - f"VM with same hostname exists in NetBox as '{existing_vm.name}' (not linked to LibreNMS)" - ) - result["can_import"] = False - elif existing_device: - logger.info(f"Found existing device by hostname: {existing_device.name}") - result["existing_device"] = existing_device - result["existing_match_type"] = "hostname" - - # Check for serial conflict on hostname-matched device - incoming_serial = libre_device.get("serial") or "" - if incoming_serial and incoming_serial != "-" and existing_device.serial != incoming_serial: - serial_conflict = ( - Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() - ) - if serial_conflict: - result["serial_action"] = "conflict" - result["serial_duplicate"] = True - result["warnings"].append( - f"Serial conflict: incoming serial '{incoming_serial}' is already assigned to " - f"device '{serial_conflict.name}' (ID: {serial_conflict.pk}) in NetBox. " - f"Investigate which device should own this serial before importing." - ) - else: - result["serial_action"] = "update_serial" - result["warnings"].append( - f"Hostname matches but serial differs (NetBox: '{existing_device.serial}', " - f"LibreNMS: '{incoming_serial}'). Hardware may have been replaced." - ) - else: - result["warnings"].append( - f"Device with same hostname exists in NetBox as '{existing_device.name}' (not linked to LibreNMS)" - ) - - result["can_import"] = False - - # Check by serial number (strong physical match - hardware identity) - if not result["existing_device"]: - serial = libre_device.get("serial") or "" - if serial and serial != "-" and not import_as_vm: - existing_by_serial = Device.objects.filter(serial=serial).first() - if existing_by_serial: - logger.info(f"Found existing device by serial: {existing_by_serial.name} (serial={serial})") - result["existing_device"] = existing_by_serial - result["existing_match_type"] = "serial" - result["can_import"] = False - - if existing_by_serial.name and existing_by_serial.name.lower() == hostname.lower(): - result["warnings"].append( - f"Device with same serial and hostname exists as '{existing_by_serial.name}' " - f"(not linked to LibreNMS)" - ) - result["serial_action"] = "link" - else: - result["warnings"].append( - f"Device with same serial ({serial}) exists as '{existing_by_serial.name}' " - f"but hostname differs (LibreNMS: '{hostname}'). Device may have been reinstalled." - ) - result["serial_action"] = "hostname_differs" - - # Check by primary IP (weaker match, IP could be reassigned) - only for devices - if not result["existing_device"]: - primary_ip = libre_device.get("ip") - if primary_ip and not import_as_vm: - from ipam.models import IPAddress - - existing_ip = IPAddress.objects.filter(address__net_host=primary_ip).first() - if existing_ip and existing_ip.assigned_object: - device = ( - existing_ip.assigned_object.device - if hasattr(existing_ip.assigned_object, "device") - else None - ) - if device: - result["existing_device"] = device - result["existing_match_type"] = "primary_ip" - result["warnings"].append( - f"IP address {primary_ip} already assigned to device '{device.name}' (not linked to LibreNMS)" - ) - result["can_import"] = False - - # Validate based on import type (Device or VM) - if import_as_vm: - # 2. For VMs: Validate Cluster (required) - Must be manually selected - from virtualization.models import Cluster - - result["cluster"]["found"] = False - result["issues"].append("Cluster must be manually selected before importing as VM") - # Provide list of available clusters for user selection (cached) - cache_key = "librenms_import_all_clusters" - all_clusters = cache.get(cache_key) - if all_clusters is None: - all_clusters = list(Cluster.objects.all()) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, all_clusters, cache_timeout) - result["cluster"]["available_clusters"] = all_clusters - - # Skip device-specific validations for VMs - result["site"]["found"] = True # Not required for VMs - result["device_type"]["found"] = True # Not required for VMs - result["device_role"]["found"] = True # Not required for VMs - - else: - # 2. For Devices: Validate Site (required) - location = libre_device.get("location", "") - site_match = find_matching_site(location) - result["site"] = site_match - - if not site_match["found"]: - result["issues"].append(f"No matching site found for location: '{location}'") - # Get alternative suggestions - if location: - all_sites = Site.objects.all()[:10] # Limit for performance - result["site"]["suggestions"] = list(all_sites) - - # 3. Validate DeviceType (required) - hardware = libre_device.get("hardware", "") - dt_match = match_librenms_hardware_to_device_type(hardware) - result["device_type"] = dt_match - - if not dt_match["matched"]: - result["issues"].append(f"No matching device type found for hardware: '{hardware}'") - # Get some device types for user to choose from - all_device_types = DeviceType.objects.all()[:10] - result["device_type"]["suggestions"] = [ - { - "device_type": dt, - "similarity": 0.0, # No fuzzy matching, just showing options - "match_field": None, - } - for dt in all_device_types - ] - else: - # Rename 'matched' to 'found' for consistency - result["device_type"]["found"] = dt_match["matched"] - result["device_type"]["device_type"] = dt_match["device_type"] - result["device_type"]["match_type"] = dt_match["match_type"] - - # 4. DeviceRole (required) - Must be manually selected by user - logger.debug(f"[{hostname}] Issues BEFORE adding role issue: {result['issues']}") - result["device_role"]["found"] = False - result["issues"].append("Device role must be manually selected before import") - logger.debug(f"[{hostname}] Issues AFTER adding role issue: {result['issues']}") - # Provide list of available roles for user selection (cached) - cache_key = "librenms_import_all_roles" - all_roles = cache.get(cache_key) - if all_roles is None: - all_roles = list(DeviceRole.objects.all()) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, all_roles, cache_timeout) - result["device_role"]["available_roles"] = all_roles - - # 4b. Rack (optional) - Provide available racks for the matched site - if site_match["found"] and site_match["site"]: - site = site_match["site"] - # Use cache to optimize rack lookups per site - cache_key = f"librenms_import_racks_site_{site.pk}" - available_racks = cache.get(cache_key) - - if available_racks is None: - from dcim.models import Rack - from django.db.models import Q - - # Query racks for this site - include both: - # 1. Racks assigned to locations within the site - # 2. Racks directly assigned to the site (without location) - available_racks = list( - Rack.objects.filter(Q(location__site=site) | Q(site=site)) - .select_related("location", "site") - .order_by("location__name", "name") - ) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, available_racks, cache_timeout) - - result["rack"]["available_racks"] = available_racks - # Rack is optional, don't add to issues - result["rack"]["found"] = True # Mark as "found" even if None (optional field) - - # Skip VM-specific validations for devices - result["cluster"]["found"] = True # Not required for devices - - # 5. Match Platform (optional - same for both devices and VMs) - os = libre_device.get("os", "") - platform_match = find_matching_platform(os) - result["platform"] = platform_match - - if not platform_match["found"] and os: - result["warnings"].append(f"No matching platform found for OS: '{os}'") - - # 6. Additional validations - if not hostname: - result["issues"].append("Device has no hostname") - - # 7. Virtual chassis detection (only for devices, not VMs) - if include_vc_detection and not import_as_vm and api is not None: - device_id = libre_device.get("device_id") - if device_id: - try: - logger.debug(f"Calling get_virtual_chassis_data for device {device_id}") - vc_detection = get_virtual_chassis_data(api, device_id, force_refresh=force_vc_refresh) - logger.debug( - f"VC detection result: is_stack={vc_detection.get('is_stack')}, " - f"member_count={vc_detection.get('member_count')}, " - f"members={len(vc_detection.get('members', []))}" - ) - if vc_detection: - result["virtual_chassis"] = vc_detection - if vc_detection["is_stack"]: - logger.debug( - f"Virtual chassis CONFIRMED for device {hostname}: " - f"{vc_detection['member_count']} members" - ) - except Exception as e: - logger.exception(f"Exception during VC detection for device {hostname}: {e}") - result["virtual_chassis"]["detection_error"] = str(e) - else: - logger.debug(f"No device_id found for {hostname}") - - # 8. Determine if device/VM is ready to import - if result["existing_device"]: - # Already matched - can_import was already set to False - result["is_ready"] = False - # Populate role from existing device so the modal shows it - existing = result["existing_device"] - if hasattr(existing, "role") and existing.role: - result["device_role"]["found"] = True - result["device_role"]["role"] = existing.role - - # Check for device type mismatch between existing device and LibreNMS - if hasattr(existing, "device_type") and existing.device_type: - librenms_dt = result["device_type"].get("device_type") - if librenms_dt and existing.device_type.pk != librenms_dt.pk: - result["device_type_mismatch"] = True - result["warnings"].append( - f"Device type mismatch: NetBox has '{existing.device_type}' " - f"but LibreNMS reports '{librenms_dt}'. " - f"This may indicate the wrong device was matched." - ) - else: - result["can_import"] = len(result["issues"]) == 0 - - if import_as_vm: - # For VMs: only cluster is required - result["is_ready"] = result["can_import"] and result["cluster"]["found"] - else: - # For Devices: site, device_type, and device_role are required - result["is_ready"] = ( - result["can_import"] - and result["site"]["found"] - and result["device_type"]["found"] - and result["device_role"]["found"] - ) - - logger.debug( - f"Validation for {libre_device.get('hostname')} ({'VM' if import_as_vm else 'Device'}): " - f"issues={len(result['issues'])}, can_import={result['can_import']}, " - f"issues_list={result['issues']}" - ) - - return result - - except Exception as e: - logger.exception(f"Error validating device for import: {libre_device.get('hostname', 'unknown')}") - result["issues"].append(f"Validation error: {str(e)}") - return result - - -def import_single_device( - device_id: int, - server_key: str = None, - validation: dict = None, - manual_mappings: dict = None, - sync_options: dict = None, - libre_device: dict = None, -) -> dict: - """ - Import a single LibreNMS device to NetBox. - - Args: - device_id: LibreNMS device ID - server_key: LibreNMS server configuration key - validation: Pre-computed validation dict (optional) - manual_mappings: Manual object mappings (optional): - - site_id: NetBox Site ID - - device_type_id: NetBox DeviceType ID - - device_role_id: NetBox DeviceRole ID - - platform_id: NetBox Platform ID (optional) - - rack_id: NetBox Rack ID (optional) - sync_options: Sync options (optional): - - sync_interfaces: bool (default True) - - sync_cables: bool (default True) - - sync_ips: bool (default True) - - sync_fields: bool (default True) - libre_device: Pre-fetched LibreNMS device data (optional). - If provided, skips API call to fetch device info. - - Returns: - dict: Import result with structure: - { - 'success': bool, - 'device': Device object or None, - 'message': str, - 'error': str or None, - 'synced': { - 'interfaces': int, - 'cables': int, - 'ip_addresses': int - } - } - """ - try: - api = LibreNMSAPI(server_key=server_key) - - # Use pre-fetched device data if provided, otherwise fetch from API - if libre_device is None: - success, libre_device = api.get_device_info(device_id) - if not success or not libre_device: - return { - "success": False, - "device": None, - "message": "", - "error": f"Failed to retrieve device {device_id} from LibreNMS", - "synced": {}, - } - - # Validate device if validation not provided - if validation is None: - validation = validate_device_for_import(libre_device) - - # Check if device already exists - if validation.get("existing_device"): - return { - "success": False, - "device": validation["existing_device"], - "message": "", - "error": f"Device already exists: {validation['existing_device'].name}", - "synced": {}, - } - - # Use validation-derived matches, allow manual mappings to override specific fields - site = validation["site"].get("site") - device_type = validation["device_type"].get("device_type") - device_role = validation["device_role"].get("role") - platform = validation["platform"].get("platform") - rack = validation.get("rack", {}).get("rack") - - if manual_mappings: - site = Site.objects.filter(id=manual_mappings.get("site_id")).first() or site - device_type = DeviceType.objects.filter(id=manual_mappings.get("device_type_id")).first() or device_type - device_role = DeviceRole.objects.filter(id=manual_mappings.get("device_role_id")).first() or device_role - - platform_id = manual_mappings.get("platform_id") - if platform_id: - from dcim.models import Platform - - platform = Platform.objects.filter(id=platform_id).first() or platform - - rack_id = manual_mappings.get("rack_id") - if rack_id: - rack = Rack.objects.select_related("location", "site").filter(id=rack_id).first() or rack - - rack = rack or validation.get("rack", {}).get("rack") - - # Validate required fields - if not site: - return { - "success": False, - "device": None, - "message": "", - "error": "Site is required but not provided", - "synced": {}, - } - if not device_type: - return { - "success": False, - "device": None, - "message": "", - "error": "Device type is required but not provided", - "synced": {}, - } - if not device_role: - return { - "success": False, - "device": None, - "message": "", - "error": "Device role is required but not provided", - "synced": {}, - } - - # Create device in NetBox - with transaction.atomic(): - # Determine device name based on sync options - use_sysname = sync_options.get("use_sysname", True) if sync_options else True - strip_domain = sync_options.get("strip_domain", False) if sync_options else False - - device_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=strip_domain, - device_id=device_id, - ) - - # Generate import timestamp comment - import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") - - device_data = { - "name": device_name, - "site": site, - "device_type": device_type, - "role": device_role, - "status": "active" if libre_device.get("status") == 1 else "offline", - "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - "custom_field_data": {"librenms_id": int(device_id)}, - } - - # Add optional fields - if platform: - device_data["platform"] = platform - - if rack: - device_data["rack"] = rack - - serial = libre_device.get("serial", "") - if serial and serial != "-": - device_data["serial"] = serial - - location_name = libre_device.get("location", "") - if location_name and location_name != "-": - from dcim.models import Location - - # Try to find matching location within the site - location = Location.objects.filter(site=site, name__iexact=location_name).first() - if location: - device_data["location"] = location - - # Create the device - device = Device(**device_data) - device.full_clean() - device.save() - - # Sync additional data based on options - sync_options = sync_options or {} - synced = {"interfaces": 0, "cables": 0, "ip_addresses": 0} - - try: - # Sync interfaces - if sync_options.get("sync_interfaces", True): - # This is simplified - would need proper request context - # For now, just log that it should be done - logger.info(f"Interface sync should be performed for device {device.name}") - - # Sync cables - if sync_options.get("sync_cables", True): - logger.info(f"Cable sync should be performed for device {device.name}") - - # Sync IP addresses - if sync_options.get("sync_ips", True): - logger.info(f"IP address sync should be performed for device {device.name}") - - except Exception as e: - logger.warning(f"Error during post-import sync: {str(e)}") - # Don't fail the import if sync fails - - return { - "success": True, - "device": device, - "message": f"Successfully imported device: {device.name}", - "error": None, - "synced": synced, - } - - except Exception as e: - logger.exception(f"Error importing device {device_id}") - return { - "success": False, - "device": None, - "message": "", - "error": str(e), - "synced": {}, - } - - -def bulk_import_devices_shared( - device_ids: List[int], - server_key: str = None, - sync_options: dict = None, - manual_mappings_per_device: dict = None, - libre_devices_cache: dict = None, - job=None, - user=None, -) -> dict: - """ - Shared function for importing multiple LibreNMS devices to NetBox. - - Used by both synchronous imports and background jobs. Handles per-device error - collection and optional progress logging when job context is provided. - - Args: - device_ids: List of LibreNMS device IDs to import - server_key: LibreNMS server configuration key - sync_options: Sync options to apply to all devices - manual_mappings_per_device: Dict mapping device_id to manual_mappings dict - Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} - libre_devices_cache: Optional dict mapping device_id to pre-fetched device data - to avoid redundant API calls. Example: {123: {...device_data...}} - job: Optional JobRunner instance for progress logging and cancellation checks - user: User performing the import (for permission checks). If job is provided, - user is extracted from job.job.user if not explicitly passed. - - Returns: - dict: Bulk import result with structure: - { - 'total': int, - 'success': List[dict], # Successfully imported devices - 'failed': List[dict], # Failed imports with errors - 'skipped': List[dict], # Skipped devices (already exist, etc.) - 'virtual_chassis_created': int # Number of VCs created - } - - Raises: - PermissionDenied: If user lacks required permissions - - Example: - >>> # Synchronous usage - >>> result = bulk_import_devices_shared([1, 2, 3, 4, 5], user=request.user) - >>> # Background job usage - >>> result = bulk_import_devices_shared([1, 2, 3], job=self) - """ - # Extract user from job if not explicitly provided - if user is None and job is not None: - user = getattr(job.job, "user", None) - - # Check permissions at start of bulk operation - required_perms = [ - "dcim.add_device", - "dcim.add_interface", - "dcim.add_virtualchassis", - ] - require_permissions(user, required_perms, "import devices") - - total = len(device_ids) - success_list = [] - failed_list = [] - skipped_list = [] - vc_created_count = 0 - processed_vc_domains = set() # Track VCs already created by domain - - # Initialize API client once for all devices to avoid repeated config parsing - api = LibreNMSAPI(server_key=server_key) - - for idx, device_id in enumerate(device_ids, start=1): - # Check for job cancellation every 5 devices - if job and idx % 5 == 0: - # Refresh job from DB to get current status - job.job.refresh_from_db() - job_status = job.job.status - status_value = job_status.value if hasattr(job_status, "value") else job_status - if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): - if job.logger: - job.logger.warning(f"Import job cancelled at device {idx} of {total}") - else: - logger.warning(f"Import cancelled at device {idx} of {total}") - break - # Log progress - if job.logger: - job.logger.info(f"Imported device {idx} of {total}") - - try: - # Use cached device data if available to avoid redundant API calls - if libre_devices_cache and device_id in libre_devices_cache: - libre_device = libre_devices_cache[device_id] - success = True - else: - success, libre_device = api.get_device_info(device_id) - - if not success or not libre_device: - error_msg = f"Failed to retrieve device {device_id} from LibreNMS" - failed_list.append({"device_id": device_id, "error": error_msg}) - if job and job.logger: - job.logger.error(error_msg) - else: - logger.error(error_msg) - continue - - validation = validate_device_for_import(libre_device, api=api) - - # Build manual mappings from validation + any provided overrides - device_mappings = {} - - # Get site and device_type from validation - if validation["site"].get("found") and validation["site"].get("site"): - device_mappings["site_id"] = validation["site"]["site"].id - if validation["device_type"].get("found") and validation["device_type"].get("device_type"): - device_mappings["device_type_id"] = validation["device_type"]["device_type"].id - if validation["platform"].get("found") and validation["platform"].get("platform"): - device_mappings["platform_id"] = validation["platform"]["platform"].id - - # Override with any manual mappings provided for this device - if manual_mappings_per_device and device_id in manual_mappings_per_device: - device_mappings.update(manual_mappings_per_device[device_id]) - - result = import_single_device( - device_id, - server_key=server_key, - sync_options=sync_options, - manual_mappings=device_mappings if device_mappings else None, - libre_device=libre_device, - ) - - if result["success"]: - success_list.append( - { - "device_id": device_id, - "device": result["device"], - "message": result["message"], - } - ) - - # Handle virtual chassis creation for stacks - vc_data = validation.get("virtual_chassis", {}) - if vc_data.get("is_stack", False): - vc_domain = f"librenms-{device_id}" - - # Only create VC if we haven't processed this stack yet - # Add to set BEFORE attempting creation to prevent race condition - if vc_domain not in processed_vc_domains: - processed_vc_domains.add(vc_domain) - try: - vc = create_virtual_chassis_with_members( - result["device"], - vc_data["members"], - libre_device, - ) - vc_created_count += 1 - log_msg = f"Created VC '{vc.name}' during bulk import for device {device_id}" - if job and job.logger: - job.logger.info(log_msg) - else: - logger.info(log_msg) - except Exception as vc_error: - # Remove from set on failure so retry is possible - processed_vc_domains.discard(vc_domain) - warn_msg = f"Failed to create VC for device {device_id}: {vc_error}" - if job and job.logger: - job.logger.warning(warn_msg) - else: - logger.warning(warn_msg) - # Don't fail the import, just log the warning - - elif result.get("device"): # Device exists - skipped_list.append({"device_id": device_id, "reason": result["error"]}) - else: # Failed to import - failed_list.append({"device_id": device_id, "error": result["error"]}) - if job and job.logger: - job.logger.error(f"Failed to import device {device_id}: {result['error']}") - - except Exception as e: - error_msg = f"Unexpected error importing device {device_id}: {str(e)}" - if job and job.logger: - job.logger.error(error_msg, exc_info=True) - else: - logger.exception(f"Unexpected error importing device {device_id}") - failed_list.append({"device_id": device_id, "error": str(e)}) - - return { - "total": total, - "success": success_list, - "failed": failed_list, - "skipped": skipped_list, - "virtual_chassis_created": vc_created_count, - } - - -def bulk_import_devices( - device_ids: List[int], - server_key: str = None, - sync_options: dict = None, - manual_mappings_per_device: dict = None, - libre_devices_cache: dict = None, - user=None, -) -> dict: - """ - Import multiple LibreNMS devices to NetBox (synchronous). - - This is the public API for synchronous imports. For background job usage, - use bulk_import_devices_shared() with a job context. - - Args: - device_ids: List of LibreNMS device IDs to import - server_key: LibreNMS server configuration key - sync_options: Sync options to apply to all devices - manual_mappings_per_device: Dict mapping device_id to manual_mappings dict - Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} - libre_devices_cache: Optional dict mapping device_id to pre-fetched device data - to avoid redundant API calls. Example: {123: {...device_data...}} - user: User performing the import (for permission checks) - - Returns: - dict: Bulk import result with structure: - { - 'total': int, - 'success': List[dict], # Successfully imported devices - 'failed': List[dict], # Failed imports with errors - 'skipped': List[dict], # Skipped devices (already exist, etc.) - 'virtual_chassis_created': int # Number of VCs created - } - - Raises: - PermissionDenied: If user lacks required permissions - """ - return bulk_import_devices_shared( - device_ids=device_ids, - server_key=server_key, - sync_options=sync_options, - manual_mappings_per_device=manual_mappings_per_device, - libre_devices_cache=libre_devices_cache, - job=None, # No job context for synchronous imports - user=user, - ) - - -def get_librenms_device_by_id(api: LibreNMSAPI, device_id: int) -> dict: - """ - Retrieve a single device from LibreNMS by ID. - - Args: - api: LibreNMSAPI instance - device_id: LibreNMS device ID - - Returns: - Device dictionary or None if not found - """ - try: - # Use the dedicated API endpoint to get device by ID - success, device = api.get_device_info(device_id) - if success and device: - return device - - logger.warning(f"Device {device_id} not found in LibreNMS") - return None - except Exception as e: - logger.exception(f"Failed to get device {device_id} from LibreNMS: {e}") - return None - - -def fetch_device_with_cache( - device_id: int, - api: LibreNMSAPI, - server_key: str = None, - libre_devices_cache: dict = None, -) -> dict | None: - """ - Fetch LibreNMS device from cache or API with automatic caching. - - Checks three sources in order: - 1. Pre-fetched cache dict (if provided) - 2. Django cache (Redis/memory) - 3. LibreNMS API (caches result for future use) - - This function consolidates the device fetching pattern used throughout - the import workflow, eliminating code duplication. - - Args: - device_id: LibreNMS device ID to fetch - api: LibreNMSAPI instance for fallback API calls - server_key: Optional server key for multi-server setups (defaults to api.server_key) - libre_devices_cache: Optional pre-fetched device cache dict - - Returns: - Device dict from LibreNMS, or None if not found - - Example: - >>> # Simple usage - >>> libre_device = fetch_device_with_cache(123, api) - >>> if libre_device: - ... print(libre_device['hostname']) - >>> - >>> # With pre-fetched cache dict - >>> cache_dict = {123: {...}, 456: {...}} - >>> libre_device = fetch_device_with_cache(123, api, libre_devices_cache=cache_dict) - """ - # Check pre-fetched cache dict first (fastest) - if libre_devices_cache and device_id in libre_devices_cache: - return libre_devices_cache[device_id] - - # Check Django cache - cache_key = get_import_device_cache_key(device_id, server_key or api.server_key) - libre_device = cache.get(cache_key) - - if not libre_device: - # Fallback to API fetch - libre_device = get_librenms_device_by_id(api, device_id) - if libre_device: - # Cache for future use - cache.set(cache_key, libre_device, timeout=api.cache_timeout) - - return libre_device - - -def create_vm_from_librenms(libre_device: dict, validation: dict, use_sysname: bool = True, role=None): - """ - Create a NetBox VirtualMachine from LibreNMS device data. - - Args: - libre_device: Device data from LibreNMS - validation: Validation result from validate_device_for_import with import_as_vm=True - use_sysname: If True, prefer sysName; if False, use hostname - role: Optional DeviceRole to assign to the VM - - Returns: - Created VirtualMachine instance - - Raises: - Exception if VM cannot be created - """ - from virtualization.models import VirtualMachine - - if not validation["can_import"]: - raise ValueError(f"VM cannot be imported: {', '.join(validation['issues'])}") - - # Extract matched objects from validation - cluster = validation["cluster"]["cluster"] - platform = validation["platform"].get("platform") - - # Determine VM name - use pre-computed name if available (handles strip_domain) - vm_name = libre_device.get("_computed_name") - if not vm_name: - vm_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=False, - device_id=libre_device.get("device_id"), - ) - - # Generate import timestamp comment - import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") - - # Create the VM with librenms_id custom field - vm = VirtualMachine.objects.create( - name=vm_name, - cluster=cluster, - role=role, # Optional VM role - platform=platform, - comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - custom_field_data={"librenms_id": int(libre_device["device_id"])}, - ) - - logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") - return vm - - -def bulk_import_vms( - vm_imports: dict[int, dict[str, int]], - api: LibreNMSAPI, - sync_options: dict = None, - libre_devices_cache: dict = None, - job=None, - user=None, -) -> dict: - """ - Import multiple LibreNMS devices as VMs in NetBox. - - Handles validation, cluster/role assignment, name determination, - and VM creation. Supports both synchronous and background job execution. - - This function consolidates VM import logic that was previously duplicated - in BulkImportDevicesView and ImportDevicesJob, ensuring consistent behavior - across synchronous and background import paths. - - Args: - vm_imports: Dict mapping device_id to {"cluster_id": int, "device_role_id": int} - api: LibreNMSAPI instance for device fetching - sync_options: Optional dict with use_sysname, strip_domain settings - libre_devices_cache: Optional pre-fetched device data cache - job: Optional JobRunner instance for background job logging/cancellation - user: User performing the import (for permission checks). If job is provided, - user is extracted from job.job.user if not explicitly passed. - - Returns: - Dict with keys: - - success: List of {"device_id": int, "device": VM, "message": str} - - failed: List of {"device_id": int, "error": str} - - skipped: List of {"device_id": int, "reason": str} - - Raises: - PermissionDenied: If user lacks required permissions - - Example: - >>> # Synchronous import from view - >>> vm_imports = {123: {"cluster_id": 5, "device_role_id": 2}} - >>> result = bulk_import_vms(vm_imports, api, sync_options, user=request.user) - >>> print(f"Created {len(result['success'])} VMs") - >>> - >>> # Background job import - >>> result = bulk_import_vms(vm_imports, api, sync_options, cache, job=self) - """ - from netbox_librenms_plugin.import_validation_helpers import ( - apply_cluster_to_validation, - apply_role_to_validation, - ) - - # Extract user from job if not explicitly provided - if user is None and job is not None: - user = getattr(job.job, "user", None) - - # Check permissions at start of bulk operation - require_permissions(user, ["virtualization.add_virtualmachine"], "import VMs") - - result = {"success": [], "failed": [], "skipped": []} - vm_ids = list(vm_imports.keys()) - - # Use job logger if available, otherwise standard logger - log = job.logger if job else logger - - for idx, vm_id in enumerate(vm_ids, start=1): - # Check for job cancellation every 5 VMs - if job and idx % 5 == 0: - job.job.refresh_from_db() - job_status = job.job.status - status_value = job_status.value if hasattr(job_status, "value") else job_status - if status_value in ("failed", "errored"): - log.warning(f"Job cancelled at VM {idx} of {len(vm_ids)}") - break - log.info(f"Imported VM {idx} of {len(vm_ids)}") - - try: - # Fetch device data (uses cache helper) - libre_device = fetch_device_with_cache(vm_id, api, api.server_key, libre_devices_cache) - - if not libre_device: - result["failed"].append( - { - "device_id": vm_id, - "error": f"Device {vm_id} not found in LibreNMS", - } - ) - log.error(f"Device {vm_id} not found in LibreNMS") - continue - - # Validate as VM - validation = validate_device_for_import(libre_device, import_as_vm=True, api=api) - - # Check if VM already exists - if validation.get("existing_device"): - result["skipped"].append( - { - "device_id": vm_id, - "reason": f"VM already exists: {validation['existing_device'].name}", - } - ) - log.info(f"VM already exists: {validation['existing_device'].name}") - continue - - # Apply manual cluster and role selections - vm_mappings = vm_imports[vm_id] - cluster_id = vm_mappings.get("cluster_id") - role_id = vm_mappings.get("device_role_id") - - if cluster_id: - cluster = Cluster.objects.filter(id=cluster_id).first() - if cluster: - apply_cluster_to_validation(validation, cluster) - - role = None - if role_id: - role = DeviceRole.objects.filter(id=role_id).first() - if role: - apply_role_to_validation(validation, role, is_vm=True) - - # Determine VM name - use_sysname = sync_options.get("use_sysname", True) if sync_options else True - strip_domain = sync_options.get("strip_domain", False) if sync_options else False - - vm_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=strip_domain, - device_id=vm_id, - ) - - # Update validation with computed name - libre_device["_computed_name"] = vm_name - - # Create VM - vm = create_vm_from_librenms(libre_device, validation, use_sysname=use_sysname, role=role) - - result["success"].append( - { - "device_id": vm_id, - "device": vm, - "message": f"VM {vm.name} created successfully", - } - ) - log.info(f"Successfully imported VM {vm.name} (ID: {vm_id})") - - except Exception as vm_error: - log.error(f"Failed to import VM {vm_id}: {vm_error}", exc_info=True) - result["failed"].append({"device_id": vm_id, "error": str(vm_error)}) - - return result - - -def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> dict: - """ - Detect if device is a stack/Virtual Chassis by analyzing ENTITY-MIB inventory. - Vendor-agnostic using standard hierarchical structure. - - Args: - api: LibreNMSAPI instance - device_id: LibreNMS device ID - - Returns: - dict with structure: - { - 'is_stack': bool, - 'member_count': int, - 'members': [ - { - 'serial': str, - 'position': int, - 'model': str, - 'name': str, - 'index': int, - 'description': str, - 'suggested_name': str # Generated using master device name - } - ] - } - Returns None if not a stack or detection fails. - - Detection Logic: - 1. Check root level (entPhysicalContainedIn=0) for parent container - 2. Find parent index (entPhysicalClass='stack' or 'chassis') - 3. Get children chassis at that parent's index - 4. If multiple chassis found → Stack detected - """ - try: - # Get the master device info to use for naming - success, device_info = api.get_device_info(device_id) - master_name = None - if success and device_info: - master_name = device_info.get("sysName") or device_info.get("hostname") - - # Step 1: Get root level items - success, root_items = api.get_inventory_filtered(device_id, ent_physical_contained_in=0) - - if not success or not root_items: - logger.debug(f"No root inventory items found for device {device_id}") - return None - - # Step 2: Find parent container index - # Could be class="stack" or the main "chassis" - parent_index = None - for item in root_items: - item_class = item.get("entPhysicalClass") - if item_class in ["stack", "chassis"]: - parent_index = item.get("entPhysicalIndex") - logger.debug(f"VC detection: Found parent container at index {parent_index} for device {device_id}") - break - - if not parent_index: - return None - - # Step 3: Get children chassis at next level - success, child_items = api.get_inventory_filtered( - device_id, - ent_physical_class="chassis", - ent_physical_contained_in=parent_index, - ) - - if not success: - return None - - # Filter for chassis only (in case API filter didn't work) - chassis_items = [item for item in (child_items or []) if item.get("entPhysicalClass") == "chassis"] - - # Step 4: Multiple chassis = stack - if len(chassis_items) <= 1: - return None - - # Step 5: Extract member info - members = [] - for idx, chassis in enumerate(chassis_items): - raw_position = chassis.get("entPhysicalParentRelPos", idx) - try: - position = int(raw_position) - except (TypeError, ValueError): - position = idx - member_data = { - "serial": chassis.get("entPhysicalSerialNum", ""), - "position": position, - "model": chassis.get("entPhysicalModelName", ""), - "name": chassis.get("entPhysicalName", ""), - "index": chassis.get("entPhysicalIndex"), - "description": chassis.get("entPhysicalDescr", ""), - } - - # Generate suggested name if we have master name - if master_name: - member_data["suggested_name"] = _generate_vc_member_name(master_name, position + 1) - else: - member_data["suggested_name"] = f"Member-{position + 1}" - - members.append(member_data) - - # Sort by position - members.sort(key=lambda m: m["position"]) - - logger.info(f"Detected stack with {len(members)} members for device {device_id}") - - return {"is_stack": True, "member_count": len(members), "members": members} - - except Exception as e: - logger.exception(f"Error detecting virtual chassis for device {device_id}: {e}") - return None - - -def _generate_vc_member_name(master_name: str, position: int, serial: str = None) -> str: - """ - Generate name for VC member device using configured pattern from settings. - - Args: - master_name: Name of the master/primary device - position: VC position number - serial: Optional serial number of the member device - - Returns: - Generated member device name - - Examples: - pattern="-M{position}" → "switch01-M2" - pattern=" ({position})" → "switch01 (2)" - pattern="-SW{position}" → "switch01-SW2" - pattern=" [{serial}]" → "switch01 [ABC123]" - """ - # Import here to avoid circular dependency - from .models import LibreNMSSettings - - # Get pattern from settings with fallback to default - try: - settings = LibreNMSSettings.objects.first() - pattern = settings.vc_member_name_pattern if settings else "-M{position}" - except Exception as e: - logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") - pattern = "-M{position}" - - # Prepare format variables - format_vars = { - "master_name": master_name, - "position": position, - "serial": serial or "", - } - - # Apply pattern - pattern should be suffix/prefix, not full name - try: - formatted_suffix = pattern.format(**format_vars) - return f"{master_name}{formatted_suffix}" - except KeyError as e: - logger.error(f"Invalid placeholder in VC naming pattern '{pattern}': {e}. Using default.") - return f"{master_name}-M{position}" - - -def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: - """ - Regenerate suggested VC member names using the actual master device name. - - This ensures preview shows accurate names after use_sysname and strip_domain - are applied to the master device name. - - Args: - vc_data: Virtual chassis detection data dict - master_name: The actual name that will be used for master device in NetBox - - Returns: - Updated vc_data dict with corrected suggested_name for each member - """ - if not vc_data or not vc_data.get("is_stack"): - return vc_data - - for idx, member in enumerate(vc_data.get("members", [])): - raw_position = member.get("position", idx) - try: - base_position = int(raw_position) - except (TypeError, ValueError): - base_position = idx - position = base_position + 1 # Convert to 1-based position - member["position"] = base_position - member["suggested_name"] = _generate_vc_member_name(master_name, position, serial=member.get("serial")) - - return vc_data - - -def create_virtual_chassis_with_members(master_device: Device, members_info: list, libre_device: dict): - """ - Create Virtual Chassis and member devices from detection info. - - This function creates a NetBox VirtualChassis with the master device - and all detected member devices, wrapped in a transaction for safety. - - Args: - master_device: The imported device (becomes VC master) - members_info: List of member dicts from VC detection - libre_device: Original LibreNMS device data - - Returns: - VirtualChassis: The created virtual chassis instance - - Raises: - ValidationError: If member count validation fails - IntegrityError: If duplicate serials/names are detected - Exception: For other creation errors - - Example members_info: - [ - {'serial': 'ABC123', 'position': 0, 'model': 'C9300-48U', 'name': 'Switch 1'}, - {'serial': 'ABC124', 'position': 1, 'model': 'C9300-48U', 'name': 'Switch 2'} - ] - """ - - # Store original master device state for rollback - original_master_name = master_device.name - original_vc = master_device.virtual_chassis - original_vc_position = master_device.vc_position - - try: - with transaction.atomic(): - # Rename master device to include position 1 pattern - master_device_new_name = _generate_vc_member_name(original_master_name, 1, serial=master_device.serial) - - # Check if renamed master conflicts with existing device - if Device.objects.filter(name=master_device_new_name).exclude(pk=master_device.pk).exists(): - logger.warning( - f"Cannot rename master to '{master_device_new_name}' - name already exists. " - f"Keeping original name '{original_master_name}'" - ) - master_base_name = original_master_name - else: - master_device.name = master_device_new_name - master_base_name = original_master_name - - # Create VC using original base name - vc_name = master_base_name - vc = VirtualChassis.objects.create( - name=vc_name, - master=master_device, - domain=f"librenms-{libre_device['device_id']}", - ) - - # Update master device - master_device.virtual_chassis = vc - master_device.vc_position = 1 # Master is position 1 - master_device.save() - - # Create member devices for remaining positions - position = 2 # Start at 2 (master is 1) - members_created = 0 - - for member in members_info: - # Skip if this is the master's serial - if member.get("serial") == master_device.serial: - continue - - serial = member.get("serial") - - member_rack = master_device.rack - member_location = master_device.location or ( - member_rack.location if member_rack and member_rack.location else None - ) - - # Check for duplicate serial - if serial and Device.objects.filter(serial=serial).exists(): - logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") - continue - - member_name = _generate_vc_member_name(master_base_name, position, serial=serial) - - # Check for duplicate name - if Device.objects.filter(name=member_name).exists(): - logger.warning(f"Device with name '{member_name}' already exists, skipping VC member creation") - continue - - Device.objects.create( - name=member_name, - device_type=master_device.device_type, - role=master_device.role, - site=master_device.site, - location=member_location, - rack=member_rack, - platform=master_device.platform, - serial=serial, - virtual_chassis=vc, - vc_position=position, - comments=f"VC member (LibreNMS: {member.get('name', 'Unknown')})\n" - f"Auto-created from stack inventory", - ) - members_created += 1 - position += 1 - - # Validate member count - expected_members = len([m for m in members_info if m.get("serial") != master_device.serial]) - if members_created < expected_members: - logger.warning( - f"Created {members_created} members but expected {expected_members}. " - "Some members may have been skipped due to duplicates." - ) - - logger.info( - f"Created Virtual Chassis '{vc.name}' with {vc.members.count()} total members " - f"(1 master + {members_created} additional)" - ) - - return vc - - except Exception as e: - # Rollback master device to original state - logger.error( - f"Virtual Chassis creation failed for device {master_device.name}: {e}. Rolling back master device changes." - ) - master_device.name = original_master_name - master_device.virtual_chassis = original_vc - master_device.vc_position = original_vc_position - master_device.save() - raise - - -def _refresh_existing_device(validation: dict) -> None: - """Refresh existing_device from DB to pick up changes made in NetBox since caching.""" - existing = validation.get("existing_device") - if not existing or not hasattr(existing, "pk"): - return - try: - from dcim.models import Device - from virtualization.models import VirtualMachine - - if validation.get("import_as_vm"): - refreshed = VirtualMachine.objects.filter(pk=existing.pk).first() - else: - refreshed = Device.objects.filter(pk=existing.pk).first() - - if refreshed: - validation["existing_device"] = refreshed - if hasattr(refreshed, "role") and refreshed.role: - validation["device_role"] = {"found": True, "role": refreshed.role} - else: - # Device was deleted since caching — recompute readiness - validation["existing_device"] = None - validation["existing_match_type"] = None - validation["can_import"] = True - if validation.get("import_as_vm"): - validation["is_ready"] = bool( - validation.get("site", {}).get("found") and validation.get("device_role", {}).get("found") - ) - else: - validation["is_ready"] = bool( - validation.get("site", {}).get("found") - and validation.get("device_type", {}).get("found") - and validation.get("device_role", {}).get("found") - ) - except Exception as e: - existing_id = getattr(existing, "pk", "unknown") if existing else "none" - logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") - - -def process_device_filters( - api: LibreNMSAPI, - filters: dict, - vc_detection_enabled: bool, - clear_cache: bool, - show_disabled: bool, - exclude_existing: bool = False, - job=None, - request=None, - return_cache_status: bool = False, -) -> List[dict] | tuple[List[dict], bool]: - """ - Process LibreNMS device filters and return validated devices. - - Shared function used by both synchronous view and background job processing. - Fetches devices, optionally pre-warms VC cache, validates each device, and - caches results for HTMX row updates. - - Args: - api: LibreNMS API client instance - filters: Filter dict with location, type, os, hostname, sysname, hardware keys - vc_detection_enabled: Whether to detect virtual chassis - clear_cache: Whether to force cache refresh - show_disabled: Whether to include disabled devices - exclude_existing: Whether to exclude devices that already exist in NetBox - job: Optional JobRunner instance for logging job events - request: Optional Django request for client disconnect detection (synchronous only) - return_cache_status: When True, returns (devices, from_cache) tuple - - Returns: - List[dict]: Validated devices with _validation key, or tuple of (devices, from_cache) - if return_cache_status is True. from_cache=True means data was loaded from existing - cache; from_cache=False means data was just fetched from LibreNMS. - """ - # Fetch devices from LibreNMS - if job: - job.logger.info(f"Fetching devices with filters: {filters}") - else: - logger.info(f"Fetching devices with filters: {filters}") - - # Always get cache status internally, even if not returning it - # We need it to determine if metadata should be updated - libre_devices, from_cache = get_librenms_devices_for_import( - api, - filters=filters, - force_refresh=clear_cache, - return_cache_status=True, - ) - - # Filter out disabled devices if requested - if not show_disabled: - libre_devices = [d for d in libre_devices if d.get("status") == 1] - - if job: - job.logger.info(f"Found {len(libre_devices)} devices to process") - else: - logger.info(f"Found {len(libre_devices)} devices") - - # Pre-warm VC cache if needed - if vc_detection_enabled and libre_devices: - device_ids = [d["device_id"] for d in libre_devices] - if job: - job.logger.info( - f"Pre-fetching virtual chassis data for {len(device_ids)} devices. This may take some time..." - ) - else: - logger.info(f"Pre-fetching VC data for {len(device_ids)} devices") - - try: - prefetch_vc_data_for_devices(api, device_ids, force_refresh=clear_cache) - if job: - job.logger.info("Virtual chassis data pre-fetch completed") - except (BrokenPipeError, ConnectionError, IOError) as e: - if request: - logger.info(f"Client disconnected during VC prefetch: {e}") - return [] - raise - - # Validate each device - validated_devices = [] - total = len(libre_devices) - api_for_validation = api if vc_detection_enabled else None - - if job: - job.logger.info(f"Starting validation of {total} devices") - # Initial check if job was already terminated before we even started - try: - from django_rq import get_queue - from rq.job import Job as RQJob - - queue = get_queue("default") - rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) - - if rq_job.is_failed or rq_job.is_stopped: - job.logger.warning("Job was already stopped before validation started") - return [] - except Exception: - # Fall back to DB check if RQ check fails - job.job.refresh_from_db() - if job.job.status == JobStatusChoices.STATUS_FAILED: - job.logger.warning("Job was stopped before validation started") - return [] - else: - logger.info(f"Validating {total} devices") - - for idx, device in enumerate(libre_devices, 1): - # Check for job termination or client disconnect periodically - if idx % 5 == 0 or idx == 1: # Check more frequently (every 5 devices + first device) - if job: - # Check if job was terminated via stop API - # CRITICAL: Check the RQ job status in Redis, not just the DB model - # NetBox's stop endpoint marks the RQ job as failed in Redis - try: - from django_rq import get_queue - from rq.job import Job as RQJob - - queue = get_queue("default") - rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) - - # Check if RQ job is in a stopped state - if rq_job.is_failed or rq_job.is_stopped: - job.logger.info( - f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." - ) - return [] - except Exception: - # If we can't check RQ status, fall back to DB status check - job.job.refresh_from_db() - if job.job.status == JobStatusChoices.STATUS_FAILED: - job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") - return [] - elif request: - # Check for client disconnect - try: - if hasattr(request, "META") and request.META.get("wsgi.input"): - pass - except (BrokenPipeError, ConnectionError, IOError): - logger.info(f"Client disconnected during validation at device {idx}") - return [] - - # Drop any cached validation/meta keys before recomputing - device.pop("_validation", None) - - # Generate shared cache key for this validated device - device_id = device["device_id"] - cache_key = get_validated_device_cache_key( - server_key=api.server_key, - filters=filters, - device_id=device_id, - vc_enabled=vc_detection_enabled, - ) - - # Check if we already have cached validation for this device - # (only if not forcing refresh) - if not clear_cache: - cached_device = cache.get(cache_key) - if cached_device: - # Use cached validation - device["_validation"] = cached_device["_validation"] - - # Refresh existing_device from DB to avoid stale data - # (user may have changed role, name, etc. in NetBox) - _refresh_existing_device(device["_validation"]) - - # Apply exclude_existing filter if enabled - if exclude_existing: - validation = device["_validation"] - if validation["existing_device"]: - continue - - validated_devices.append(device) - continue - - # Not in cache or forcing refresh - validate now - try: - validation = validate_device_for_import( - device, - api=api_for_validation, - include_vc_detection=vc_detection_enabled, - force_vc_refresh=clear_cache, - ) - except (BrokenPipeError, ConnectionError, IOError) as e: - if request: - logger.info(f"Client disconnected during device validation: {e}") - return [] - raise - - # Set VC detection metadata - if not vc_detection_enabled: - validation["virtual_chassis"] = empty_virtual_chassis_data() - - # Apply exclude_existing filter if enabled - if exclude_existing and validation["existing_device"]: - continue - - device["_validation"] = validation - validated_devices.append(device) - - # Cache with TWO keys for different purposes: - # 1. Complex key (with filter context) - for full validated device with all metadata - cache.set(cache_key, device, timeout=api.cache_timeout) - - # 2. Simple key (device ID only) - for quick device data lookup by role/rack updates - # This avoids redundant API calls when user interacts with dropdowns - simple_cache_key = get_import_device_cache_key(device_id, api.server_key) - # Cache just the raw device data (not the full validation result) - # This is what get_validated_device_with_selections() expects - device_data_only = {k: v for k, v in device.items() if k != "_validation"} - cache.set(simple_cache_key, device_data_only, timeout=api.cache_timeout) - - # Store cache metadata (timestamp) for all filter operations - # This enables countdown display regardless of background job vs synchronous execution - # Always store metadata when we have validated devices, even if from_cache - # This ensures metadata is available for countdown display - if validated_devices: - from datetime import datetime, timezone - - cache_metadata_key = get_cache_metadata_key( - server_key=api.server_key, filters=filters, vc_enabled=vc_detection_enabled - ) - - # Check if metadata already exists to preserve original timestamp - # BUT: if clear_cache was requested or data came fresh from LibreNMS, update it - existing_metadata = cache.get(cache_metadata_key) - should_update = clear_cache or not from_cache - - if existing_metadata and not should_update: - # Metadata exists and cache wasn't cleared, keep using it (preserves original cache time) - pass - else: - # No metadata exists, OR cache was cleared, OR fresh data - create/update it now - cache_metadata = { - "cached_at": datetime.now(timezone.utc).isoformat(), - "cache_timeout": api.cache_timeout, - "filters": filters, - "vc_enabled": vc_detection_enabled, - "device_count": len(validated_devices), - } - cache.set(cache_metadata_key, cache_metadata, timeout=api.cache_timeout) - - # Maintain cache index for this server to enable listing active searches - cache_index_key = f"librenms_cache_index_{api.server_key}" - cache_index = cache.get(cache_index_key, []) - # Add this cache key if not already in index - if cache_metadata_key not in cache_index: - cache_index.append(cache_metadata_key) - # Store index with same timeout as the metadata - cache.set(cache_index_key, cache_index, timeout=api.cache_timeout) - - if job: - if exclude_existing: - filtered_count = total - len(validated_devices) - job.logger.info( - f"Validation complete: {len(validated_devices)} devices passed filter, " - f"{filtered_count} filtered out (existing devices excluded)" - ) - else: - job.logger.info(f"Validation complete: {len(validated_devices)} devices ready for import") - else: - logger.info(f"Processed {len(validated_devices)} validated devices") - - if return_cache_status: - return validated_devices, from_cache - return validated_devices diff --git a/netbox_librenms_plugin/import_utils/__init__.py b/netbox_librenms_plugin/import_utils/__init__.py new file mode 100644 index 0000000000..b7a08fef59 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/__init__.py @@ -0,0 +1,52 @@ +""" +Utilities for importing devices from LibreNMS to NetBox. + +This package provides functions for: +- Validating LibreNMS devices for import +- Retrieving filtered LibreNMS devices +- Importing single and multiple devices +- Smart matching of NetBox objects +- Permission checking for import operations +- Virtual chassis detection and creation + +All imports below are intentional re-exports so that existing callers +can continue using ``from netbox_librenms_plugin.import_utils import X``. +The F401 suppressions prevent linters from flagging them as unused. +""" + +from .bulk_import import ( # noqa: F401 + bulk_import_devices, + bulk_import_devices_shared, + process_device_filters, +) +from .cache import ( # noqa: F401 + get_active_cached_searches, + get_cache_metadata_key, + get_import_device_cache_key, + get_validated_device_cache_key, +) +from .device_operations import ( # noqa: F401 + _determine_device_name, + fetch_device_with_cache, + get_librenms_device_by_id, + import_single_device, + validate_device_for_import, +) +from .filters import ( # noqa: F401 + _apply_client_filters, + get_device_count_for_filters, + get_librenms_devices_for_import, +) +from .permissions import check_user_permissions, require_permissions # noqa: F401 +from .virtual_chassis import ( # noqa: F401 + _clone_virtual_chassis_data, + _generate_vc_member_name, + _vc_cache_key, + create_virtual_chassis_with_members, + detect_virtual_chassis_from_inventory, + empty_virtual_chassis_data, + get_virtual_chassis_data, + prefetch_vc_data_for_devices, + update_vc_member_suggested_names, +) +from .vm_operations import bulk_import_vms, create_vm_from_librenms # noqa: F401 diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py new file mode 100644 index 0000000000..8f55e0b6cf --- /dev/null +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -0,0 +1,563 @@ +"""Bulk import orchestration for devices and filter processing.""" + +import logging +from typing import List + +from core.choices import JobStatusChoices +from django.core.cache import cache + +from ..librenms_api import LibreNMSAPI +from .cache import get_cache_metadata_key, get_import_device_cache_key, get_validated_device_cache_key +from .device_operations import import_single_device, validate_device_for_import +from .filters import get_librenms_devices_for_import +from .permissions import require_permissions +from .virtual_chassis import ( + create_virtual_chassis_with_members, + empty_virtual_chassis_data, + prefetch_vc_data_for_devices, +) + +logger = logging.getLogger(__name__) + + +def bulk_import_devices_shared( + device_ids: List[int], + server_key: str = None, + sync_options: dict = None, + manual_mappings_per_device: dict = None, + libre_devices_cache: dict = None, + job=None, + user=None, +) -> dict: + """ + Shared function for importing multiple LibreNMS devices to NetBox. + + Used by both synchronous imports and background jobs. Handles per-device error + collection and optional progress logging when job context is provided. + + Args: + device_ids: List of LibreNMS device IDs to import + server_key: LibreNMS server configuration key + sync_options: Sync options to apply to all devices + manual_mappings_per_device: Dict mapping device_id to manual_mappings dict + Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} + libre_devices_cache: Optional dict mapping device_id to pre-fetched device data + to avoid redundant API calls. Example: {123: {...device_data...}} + job: Optional JobRunner instance for progress logging and cancellation checks + user: User performing the import (for permission checks). If job is provided, + user is extracted from job.job.user if not explicitly passed. + + Returns: + dict: Bulk import result with structure: + { + 'total': int, + 'success': List[dict], # Successfully imported devices + 'failed': List[dict], # Failed imports with errors + 'skipped': List[dict], # Skipped devices (already exist, etc.) + 'virtual_chassis_created': int # Number of VCs created + } + + Raises: + PermissionDenied: If user lacks required permissions + + Example: + >>> # Synchronous usage + >>> result = bulk_import_devices_shared([1, 2, 3, 4, 5], user=request.user) + >>> # Background job usage + >>> result = bulk_import_devices_shared([1, 2, 3], job=self) + """ + # Extract user from job if not explicitly provided + if user is None and job is not None: + user = getattr(job.job, "user", None) + + # Check permissions at start of bulk operation + required_perms = [ + "dcim.add_device", + "dcim.add_interface", + "dcim.add_virtualchassis", + ] + require_permissions(user, required_perms, "import devices") + + total = len(device_ids) + success_list = [] + failed_list = [] + skipped_list = [] + vc_created_count = 0 + processed_vc_domains = set() # Track VCs already created by domain + + # Initialize API client once for all devices to avoid repeated config parsing + api = LibreNMSAPI(server_key=server_key) + + for idx, device_id in enumerate(device_ids, start=1): + # Check for job cancellation every 5 devices + if job and idx % 5 == 0: + # Refresh job from DB to get current status + job.job.refresh_from_db() + job_status = job.job.status + status_value = job_status.value if hasattr(job_status, "value") else job_status + if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): + if job.logger: + job.logger.warning(f"Import job cancelled at device {idx} of {total}") + else: + logger.warning(f"Import cancelled at device {idx} of {total}") + break + # Log progress + if job.logger: + job.logger.info(f"Imported device {idx} of {total}") + + try: + # Use cached device data if available to avoid redundant API calls + if libre_devices_cache and device_id in libre_devices_cache: + libre_device = libre_devices_cache[device_id] + success = True + else: + success, libre_device = api.get_device_info(device_id) + + if not success or not libre_device: + error_msg = f"Failed to retrieve device {device_id} from LibreNMS" + failed_list.append({"device_id": device_id, "error": error_msg}) + if job and job.logger: + job.logger.error(error_msg) + else: + logger.error(error_msg) + continue + + validation = validate_device_for_import(libre_device, api=api) + + # Build manual mappings from validation + any provided overrides + device_mappings = {} + + # Get site and device_type from validation + if validation["site"].get("found") and validation["site"].get("site"): + device_mappings["site_id"] = validation["site"]["site"].id + if validation["device_type"].get("found") and validation["device_type"].get("device_type"): + device_mappings["device_type_id"] = validation["device_type"]["device_type"].id + if validation["platform"].get("found") and validation["platform"].get("platform"): + device_mappings["platform_id"] = validation["platform"]["platform"].id + + # Override with any manual mappings provided for this device + if manual_mappings_per_device and device_id in manual_mappings_per_device: + device_mappings.update(manual_mappings_per_device[device_id]) + + result = import_single_device( + device_id, + server_key=server_key, + sync_options=sync_options, + manual_mappings=device_mappings if device_mappings else None, + libre_device=libre_device, + ) + + if result["success"]: + success_list.append( + { + "device_id": device_id, + "device": result["device"], + "message": result["message"], + } + ) + + # Handle virtual chassis creation for stacks + vc_data = validation.get("virtual_chassis", {}) + if vc_data.get("is_stack", False): + vc_domain = f"librenms-{device_id}" + + # Only create VC if we haven't processed this stack yet + # Add to set BEFORE attempting creation to prevent race condition + if vc_domain not in processed_vc_domains: + processed_vc_domains.add(vc_domain) + try: + vc = create_virtual_chassis_with_members( + result["device"], + vc_data["members"], + libre_device, + ) + vc_created_count += 1 + log_msg = f"Created VC '{vc.name}' during bulk import for device {device_id}" + if job and job.logger: + job.logger.info(log_msg) + else: + logger.info(log_msg) + except Exception as vc_error: + # Remove from set on failure so retry is possible + processed_vc_domains.discard(vc_domain) + warn_msg = f"Failed to create VC for device {device_id}: {vc_error}" + if job and job.logger: + job.logger.warning(warn_msg) + else: + logger.warning(warn_msg) + # Don't fail the import, just log the warning + + elif result.get("device"): # Device exists + skipped_list.append({"device_id": device_id, "reason": result["error"]}) + else: # Failed to import + failed_list.append({"device_id": device_id, "error": result["error"]}) + if job and job.logger: + job.logger.error(f"Failed to import device {device_id}: {result['error']}") + + except Exception as e: + error_msg = f"Unexpected error importing device {device_id}: {str(e)}" + if job and job.logger: + job.logger.error(error_msg, exc_info=True) + else: + logger.exception(f"Unexpected error importing device {device_id}") + failed_list.append({"device_id": device_id, "error": str(e)}) + + return { + "total": total, + "success": success_list, + "failed": failed_list, + "skipped": skipped_list, + "virtual_chassis_created": vc_created_count, + } + + +def bulk_import_devices( + device_ids: List[int], + server_key: str = None, + sync_options: dict = None, + manual_mappings_per_device: dict = None, + libre_devices_cache: dict = None, + user=None, +) -> dict: + """ + Import multiple LibreNMS devices to NetBox (synchronous). + + This is the public API for synchronous imports. For background job usage, + use bulk_import_devices_shared() with a job context. + + Args: + device_ids: List of LibreNMS device IDs to import + server_key: LibreNMS server configuration key + sync_options: Sync options to apply to all devices + manual_mappings_per_device: Dict mapping device_id to manual_mappings dict + Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} + libre_devices_cache: Optional dict mapping device_id to pre-fetched device data + to avoid redundant API calls. Example: {123: {...device_data...}} + user: User performing the import (for permission checks) + + Returns: + dict: Bulk import result with structure: + { + 'total': int, + 'success': List[dict], # Successfully imported devices + 'failed': List[dict], # Failed imports with errors + 'skipped': List[dict], # Skipped devices (already exist, etc.) + 'virtual_chassis_created': int # Number of VCs created + } + + Raises: + PermissionDenied: If user lacks required permissions + """ + return bulk_import_devices_shared( + device_ids=device_ids, + server_key=server_key, + sync_options=sync_options, + manual_mappings_per_device=manual_mappings_per_device, + libre_devices_cache=libre_devices_cache, + job=None, # No job context for synchronous imports + user=user, + ) + + +def _refresh_existing_device(validation: dict) -> None: + """Refresh existing_device from DB to pick up changes made in NetBox since caching.""" + existing = validation.get("existing_device") + if not existing or not hasattr(existing, "pk"): + return + try: + from dcim.models import Device + from virtualization.models import VirtualMachine + + if validation.get("import_as_vm"): + refreshed = VirtualMachine.objects.filter(pk=existing.pk).first() + else: + refreshed = Device.objects.filter(pk=existing.pk).first() + + if refreshed: + validation["existing_device"] = refreshed + if hasattr(refreshed, "role") and refreshed.role: + validation["device_role"] = {"found": True, "role": refreshed.role} + else: + # Device was deleted since caching — recompute readiness + validation["existing_device"] = None + validation["existing_match_type"] = None + validation["can_import"] = True + if validation.get("import_as_vm"): + validation["is_ready"] = bool( + validation.get("site", {}).get("found") and validation.get("device_role", {}).get("found") + ) + else: + validation["is_ready"] = bool( + validation.get("site", {}).get("found") + and validation.get("device_type", {}).get("found") + and validation.get("device_role", {}).get("found") + ) + except Exception as e: + existing_id = getattr(existing, "pk", "unknown") if existing else "none" + logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") + + +def process_device_filters( + api: LibreNMSAPI, + filters: dict, + vc_detection_enabled: bool, + clear_cache: bool, + show_disabled: bool, + exclude_existing: bool = False, + job=None, + request=None, + return_cache_status: bool = False, +) -> List[dict] | tuple[List[dict], bool]: + """ + Process LibreNMS device filters and return validated devices. + + Shared function used by both synchronous view and background job processing. + Fetches devices, optionally pre-warms VC cache, validates each device, and + caches results for HTMX row updates. + + Args: + api: LibreNMS API client instance + filters: Filter dict with location, type, os, hostname, sysname, hardware keys + vc_detection_enabled: Whether to detect virtual chassis + clear_cache: Whether to force cache refresh + show_disabled: Whether to include disabled devices + exclude_existing: Whether to exclude devices that already exist in NetBox + job: Optional JobRunner instance for logging job events + request: Optional Django request for client disconnect detection (synchronous only) + return_cache_status: When True, returns (devices, from_cache) tuple + + Returns: + List[dict]: Validated devices with _validation key, or tuple of (devices, from_cache) + if return_cache_status is True. from_cache=True means data was loaded from existing + cache; from_cache=False means data was just fetched from LibreNMS. + """ + # Fetch devices from LibreNMS + if job: + job.logger.info(f"Fetching devices with filters: {filters}") + else: + logger.info(f"Fetching devices with filters: {filters}") + + # Always get cache status internally, even if not returning it + # We need it to determine if metadata should be updated + libre_devices, from_cache = get_librenms_devices_for_import( + api, + filters=filters, + force_refresh=clear_cache, + return_cache_status=True, + ) + + # Filter out disabled devices if requested + if not show_disabled: + libre_devices = [d for d in libre_devices if d.get("status") == 1] + + if job: + job.logger.info(f"Found {len(libre_devices)} devices to process") + else: + logger.info(f"Found {len(libre_devices)} devices") + + # Pre-warm VC cache if needed + if vc_detection_enabled and libre_devices: + device_ids = [d["device_id"] for d in libre_devices] + if job: + job.logger.info( + f"Pre-fetching virtual chassis data for {len(device_ids)} devices. This may take some time..." + ) + else: + logger.info(f"Pre-fetching VC data for {len(device_ids)} devices") + + try: + prefetch_vc_data_for_devices(api, device_ids, force_refresh=clear_cache) + if job: + job.logger.info("Virtual chassis data pre-fetch completed") + except (BrokenPipeError, ConnectionError, IOError) as e: + if request: + logger.info(f"Client disconnected during VC prefetch: {e}") + return [] + raise + + # Validate each device + validated_devices = [] + total = len(libre_devices) + api_for_validation = api if vc_detection_enabled else None + + if job: + job.logger.info(f"Starting validation of {total} devices") + # Initial check if job was already terminated before we even started + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + + if rq_job.is_failed or rq_job.is_stopped: + job.logger.warning("Job was already stopped before validation started") + return [] + except Exception: + # Fall back to DB check if RQ check fails + job.job.refresh_from_db() + if job.job.status == JobStatusChoices.STATUS_FAILED: + job.logger.warning("Job was stopped before validation started") + return [] + else: + logger.info(f"Validating {total} devices") + + for idx, device in enumerate(libre_devices, 1): + # Check for job termination or client disconnect periodically + if idx % 5 == 0 or idx == 1: # Check more frequently (every 5 devices + first device) + if job: + # Check if job was terminated via stop API + # CRITICAL: Check the RQ job status in Redis, not just the DB model + # NetBox's stop endpoint marks the RQ job as failed in Redis + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + + # Check if RQ job is in a stopped state + if rq_job.is_failed or rq_job.is_stopped: + job.logger.info( + f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." + ) + return [] + except Exception: + # If we can't check RQ status, fall back to DB status check + job.job.refresh_from_db() + if job.job.status == JobStatusChoices.STATUS_FAILED: + job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") + return [] + elif request: + # Check for client disconnect + try: + if hasattr(request, "META") and request.META.get("wsgi.input"): + pass + except (BrokenPipeError, ConnectionError, IOError): + logger.info(f"Client disconnected during validation at device {idx}") + return [] + + # Drop any cached validation/meta keys before recomputing + device.pop("_validation", None) + + # Generate shared cache key for this validated device + device_id = device["device_id"] + cache_key = get_validated_device_cache_key( + server_key=api.server_key, + filters=filters, + device_id=device_id, + vc_enabled=vc_detection_enabled, + ) + + # Check if we already have cached validation for this device + # (only if not forcing refresh) + if not clear_cache: + cached_device = cache.get(cache_key) + if cached_device: + # Use cached validation + device["_validation"] = cached_device["_validation"] + + # Refresh existing_device from DB to avoid stale data + # (user may have changed role, name, etc. in NetBox) + _refresh_existing_device(device["_validation"]) + + # Apply exclude_existing filter if enabled + if exclude_existing: + validation = device["_validation"] + if validation["existing_device"]: + continue + + validated_devices.append(device) + continue + + # Not in cache or forcing refresh - validate now + try: + validation = validate_device_for_import( + device, + api=api_for_validation, + include_vc_detection=vc_detection_enabled, + force_vc_refresh=clear_cache, + ) + except (BrokenPipeError, ConnectionError, IOError) as e: + if request: + logger.info(f"Client disconnected during device validation: {e}") + return [] + raise + + # Set VC detection metadata + if not vc_detection_enabled: + validation["virtual_chassis"] = empty_virtual_chassis_data() + + # Apply exclude_existing filter if enabled + if exclude_existing and validation["existing_device"]: + continue + + device["_validation"] = validation + validated_devices.append(device) + + # Cache with TWO keys for different purposes: + # 1. Complex key (with filter context) - for full validated device with all metadata + cache.set(cache_key, device, timeout=api.cache_timeout) + + # 2. Simple key (device ID only) - for quick device data lookup by role/rack updates + # This avoids redundant API calls when user interacts with dropdowns + simple_cache_key = get_import_device_cache_key(device_id, api.server_key) + # Cache just the raw device data (not the full validation result) + # This is what get_validated_device_with_selections() expects + device_data_only = {k: v for k, v in device.items() if k != "_validation"} + cache.set(simple_cache_key, device_data_only, timeout=api.cache_timeout) + + # Store cache metadata (timestamp) for all filter operations + # This enables countdown display regardless of background job vs synchronous execution + # Always store metadata when we have validated devices, even if from_cache + # This ensures metadata is available for countdown display + if validated_devices: + from datetime import datetime, timezone + + cache_metadata_key = get_cache_metadata_key( + server_key=api.server_key, filters=filters, vc_enabled=vc_detection_enabled + ) + + # Check if metadata already exists to preserve original timestamp + # BUT: if clear_cache was requested or data came fresh from LibreNMS, update it + existing_metadata = cache.get(cache_metadata_key) + should_update = clear_cache or not from_cache + + if existing_metadata and not should_update: + # Metadata exists and cache wasn't cleared, keep using it (preserves original cache time) + pass + else: + # No metadata exists, OR cache was cleared, OR fresh data - create/update it now + cache_metadata = { + "cached_at": datetime.now(timezone.utc).isoformat(), + "cache_timeout": api.cache_timeout, + "filters": filters, + "vc_enabled": vc_detection_enabled, + "device_count": len(validated_devices), + } + cache.set(cache_metadata_key, cache_metadata, timeout=api.cache_timeout) + + # Maintain cache index for this server to enable listing active searches + cache_index_key = f"librenms_cache_index_{api.server_key}" + cache_index = cache.get(cache_index_key, []) + # Add this cache key if not already in index + if cache_metadata_key not in cache_index: + cache_index.append(cache_metadata_key) + # Store index with same timeout as the metadata + cache.set(cache_index_key, cache_index, timeout=api.cache_timeout) + + if job: + if exclude_existing: + filtered_count = total - len(validated_devices) + job.logger.info( + f"Validation complete: {len(validated_devices)} devices passed filter, " + f"{filtered_count} filtered out (existing devices excluded)" + ) + else: + job.logger.info(f"Validation complete: {len(validated_devices)} devices ready for import") + else: + logger.info(f"Processed {len(validated_devices)} validated devices") + + if return_cache_status: + return validated_devices, from_cache + return validated_devices diff --git a/netbox_librenms_plugin/import_utils/cache.py b/netbox_librenms_plugin/import_utils/cache.py new file mode 100644 index 0000000000..716e9dc6a1 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/cache.py @@ -0,0 +1,158 @@ +"""Cache key generation and management for device import operations.""" + +import logging + +from django.core.cache import cache + +logger = logging.getLogger(__name__) + + +def get_cache_metadata_key(server_key: str, filters: dict, vc_enabled: bool) -> str: + """ + Generate a consistent cache metadata key from filter parameters. + + Args: + server_key: LibreNMS server identifier + filters: Filter dictionary + vc_enabled: Whether VC detection is enabled + + Returns: + str: Consistent cache key for metadata + """ + # Sort filter items to ensure consistent key generation + filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v) + return f"librenms_filter_cache_metadata_{server_key}_{filter_parts}_{vc_enabled}" + + +def get_active_cached_searches(server_key: str) -> list[dict]: + """ + Retrieve all active cached searches for a server and enrich with display-friendly values. + + Enriches raw filter IDs with human-readable names by looking up location names + from cached choices and converting type codes to display names. + + Args: + server_key: LibreNMS server identifier + + Returns: + List of dicts containing cache metadata with enriched display_filters + """ + from datetime import datetime, timezone + + cache_index_key = f"librenms_cache_index_{server_key}" + cache_index = cache.get(cache_index_key, []) + + active_searches = [] + valid_cache_keys = [] + + # Get location and type choices for enriching display + location_choices = {} + type_choices = { + "": "All Types", + "network": "Network", + "server": "Server", + "storage": "Storage", + "wireless": "Wireless", + "firewall": "Firewall", + "power": "Power", + "appliance": "Appliance", + "printer": "Printer", + "loadbalancer": "Load Balancer", + "other": "Other", + } + + # Get cached location choices for enrichment + location_cache_key = "librenms_locations_choices" + cached_locations = cache.get(location_cache_key) + if cached_locations: + location_choices = dict(cached_locations) + + for cache_key in cache_index: + metadata = cache.get(cache_key) + if metadata: + # Cache still exists, calculate time remaining + cached_at = datetime.fromisoformat(metadata.get("cached_at")) + cache_timeout = metadata.get("cache_timeout", 300) + now = datetime.now(timezone.utc) + age_seconds = (now - cached_at).total_seconds() + remaining_seconds = max(0, cache_timeout - age_seconds) + + if remaining_seconds > 0: + # Add remaining time and cache key + metadata["remaining_seconds"] = int(remaining_seconds) + metadata["cache_key"] = cache_key + + # Enrich filters with human-readable display values + if "filters" in metadata: + display_filters = metadata["filters"].copy() + # Convert location ID to location name + if "location" in display_filters and display_filters["location"] in location_choices: + display_filters["location"] = location_choices[display_filters["location"]] + # Convert type code to display name + if "type" in display_filters and display_filters["type"] in type_choices: + display_filters["type"] = type_choices[display_filters["type"]] + metadata["display_filters"] = display_filters + else: + # Fallback if filters key missing + metadata["display_filters"] = {} + + active_searches.append(metadata) + valid_cache_keys.append(cache_key) + + # Clean up index if any keys have expired + if len(valid_cache_keys) < len(cache_index): + cache.set(cache_index_key, valid_cache_keys, timeout=3600) + + # Sort by most recent first + active_searches.sort(key=lambda x: x.get("cached_at", ""), reverse=True) + + return active_searches + + +def get_validated_device_cache_key(server_key: str, filters: dict, device_id: int | str, vc_enabled: bool) -> str: + """ + Generate a consistent cache key for validated device data. + + This ensures both synchronous and background job processing use the same + cache keys, avoiding duplicate validation work and cache entries. + + Args: + server_key: LibreNMS server key + filters: Filter dict with location, type, os, hostname, sysname, hardware keys + device_id: LibreNMS device ID + vc_enabled: Whether virtual chassis detection was enabled + + Returns: + str: Cache key for the validated device + + Example: + >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) + >>> key + 'validated_device_default_-1234567890_123_vc' + """ + # Sort filters for consistent hashing + filter_hash = hash(str(sorted(filters.items()))) + vc_part = "vc" if vc_enabled else "novc" + return f"validated_device_{server_key}_{filter_hash}_{device_id}_{vc_part}" + + +def get_import_device_cache_key(device_id: int | str, server_key: str = "default") -> str: + """ + Generate cache key for raw LibreNMS device data. + + This key is used to cache raw device data (without validation metadata) + to avoid redundant API calls when users interact with dropdowns during + the import workflow. + + Args: + device_id: LibreNMS device ID + server_key: LibreNMS server identifier for multi-server setups + + Returns: + str: Cache key for the device data + + Example: + >>> get_import_device_cache_key(123, "production") + 'import_device_data_production_123' + """ + return f"import_device_data_{server_key}_{device_id}" diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py new file mode 100644 index 0000000000..539e133c0b --- /dev/null +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -0,0 +1,859 @@ +"""Device validation, import, and fetch operations.""" + +import logging + +from dcim.models import Device, DeviceRole, DeviceType, Rack, Site +from django.core.cache import cache +from virtualization.models import Cluster +from django.db import transaction +from django.utils import timezone + +from ..librenms_api import LibreNMSAPI +from ..utils import ( + find_matching_platform, + find_matching_site, + match_librenms_hardware_to_device_type, +) +from .cache import get_import_device_cache_key +from .virtual_chassis import empty_virtual_chassis_data, get_virtual_chassis_data + +logger = logging.getLogger(__name__) + + +def _determine_device_name( + libre_device: dict, + use_sysname: bool = True, + strip_domain: bool = False, + device_id: int | str = None, +) -> str: + """ + Determine the device/VM name from LibreNMS data. + + Centralized logic for building device names with consistent handling of: + - sysName vs hostname preference + - Domain stripping (avoiding IP addresses) + - Fallback to device_id when name is missing + + Args: + libre_device: Device data from LibreNMS + use_sysname: If True, prefer sysName; if False, use hostname + strip_domain: If True, strip domain suffix (e.g., '.example.com') + device_id: LibreNMS device ID for fallback name generation + + Returns: + str: The determined device name + + Example: + >>> _determine_device_name({'sysName': 'router.example.com', 'hostname': 'router'}, + ... use_sysname=True, strip_domain=True) + 'router' + """ + # Determine base name based on use_sysname preference + if use_sysname: + name = libre_device.get("sysName") or libre_device.get("hostname") + else: + name = libre_device.get("hostname") or libre_device.get("sysName") + + # Fallback to device_id if no name found + if not name: + if device_id is not None: + name = f"device-{device_id}" + else: + name = libre_device.get("device_id", "unknown") + name = f"device-{name}" + + # Strip domain if requested (but not for IP addresses) + if strip_domain and name and "." in name: + try: + from ipaddress import ip_address + + ip_address(name) + # It's a valid IP address, don't strip + except ValueError: + # Not an IP, safe to strip domain + name = name.split(".")[0] + + return name + + +def validate_device_for_import( + libre_device: dict, + import_as_vm: bool = False, + api: "LibreNMSAPI" = None, + *, + include_vc_detection: bool = True, + force_vc_refresh: bool = False, +) -> dict: + """ + Validate if a LibreNMS device can be imported to NetBox. + + Performs comprehensive validation: + - Checks if device already exists in NetBox + - Validates required prerequisites (Site, DeviceType, DeviceRole for devices) + OR (Cluster for VMs) + - Provides smart matching for missing objects + - Detects virtual chassis/stack configuration (if API provided) + - Returns detailed validation status + + Args: + libre_device: Device data from LibreNMS + import_as_vm: If True, validate for VM import instead of device import + api: Optional LibreNMSAPI instance for virtual chassis detection + include_vc_detection: Skip VC detection when False to speed up bulk operations + force_vc_refresh: When True, bypass cached VC data and re-query LibreNMS + + Returns: + dict: Validation result with structure: + { + 'is_ready': bool, # Can import without user intervention + 'can_import': bool, # Can import (possibly after configuration) + 'import_as_vm': bool, # Whether importing as VM + 'existing_device': Device or VirtualMachine or None, + 'issues': List[str], # Blocking issues + 'warnings': List[str], # Non-blocking warnings + 'site': { # Only for devices + 'found': bool, + 'site': Site or None, + 'match_type': str, # 'exact' or None + 'suggestions': List[Site] # Alternative suggestions + }, + 'device_type': { # Only for devices + 'found': bool, + 'device_type': DeviceType or None, + 'match_type': str, # 'exact' or None + 'suggestions': List[dict] # Device types for user selection + }, + 'device_role': { # Only for devices + 'found': bool, # Always False - requires manual selection + 'role': DeviceRole or None, + 'available_roles': List[DeviceRole] # All roles for user selection + }, + 'cluster': { # Only for VMs + 'found': bool, # Always False - requires manual selection + 'cluster': Cluster or None, + 'available_clusters': List[Cluster] # All clusters for user selection + }, + 'platform': { + 'found': bool, + 'platform': Platform or None, + 'match_type': str # 'exact' or None + } + } + + Example: + >>> validation = validate_device_for_import(libre_device) + >>> if validation['is_ready']: + ... import_single_device(libre_device['device_id']) + """ + result = { + "is_ready": False, + "can_import": False, + "import_as_vm": import_as_vm, + "existing_device": None, + "existing_match_type": None, # Track how existing device was matched + "serial_action": None, # None, "link", "conflict", "update_serial", "hostname_differs" + "serial_confirmed": False, # True when librenms_id match and serial matches + "serial_duplicate": False, # True when incoming serial is already on a different device + "name_matches": False, # True when existing device name matches LibreNMS sysName + "name_sync_available": False, # True when existing device name differs from sysName + "suggested_name": None, # sysName to suggest when name_sync_available is True + "device_type_mismatch": False, # True when existing device's type differs from LibreNMS + "issues": [], + "warnings": [], + "virtual_chassis": empty_virtual_chassis_data(), + "site": { + "found": False, + "site": None, + "match_type": None, + "suggestions": [], + }, + "device_type": { + "found": False, + "device_type": None, + "match_type": None, + "suggestions": [], + }, + "device_role": { + "found": False, + "role": None, + "available_roles": [], + }, + "cluster": { + "found": False, + "cluster": None, + "available_clusters": [], + }, + "platform": {"found": False, "platform": None, "match_type": None}, + "rack": { + "found": False, + "rack": None, + "available_racks": [], + }, + } + + try: + # 1. Check if device/VM already exists in NetBox + # Always check both Devices AND VMs to properly detect existing objects + librenms_id = libre_device.get("device_id") + hostname = libre_device.get("hostname", "") + logger.debug( + f"Checking for existing device/VM: " + f"librenms_id={librenms_id} (type={type(librenms_id).__name__}), " + f"hostname={hostname}" + ) + + from virtualization.models import VirtualMachine + + # Check for existing VM first (by librenms_id custom field) + # Always query with int to match custom field type + try: + existing_vm = VirtualMachine.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() + except (ValueError, TypeError): + # librenms_id is not convertible to int; no match will be found + existing_vm = None + + if existing_vm: + logger.info(f"Found existing VM: {existing_vm.name} (matched by librenms_id={librenms_id})") + result["existing_device"] = existing_vm + result["existing_match_type"] = "librenms_id" + result["import_as_vm"] = True # Force VM mode since VM exists + result["can_import"] = False + + # Check if name matches sysName + # Note: name_sync_available/suggested_name are intentionally not set for VMs + # because UpdateDeviceNameView only supports Device objects; VM name-sync + # would require a separate implementation. + sys_name = libre_device.get("sysName") or "" + if sys_name and existing_vm.name == sys_name: + result["name_matches"] = True + + # Check for existing Device (by librenms_id custom field) + # Always query with int to match custom field type + if not result["existing_device"]: + try: + existing_device = Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() + except (ValueError, TypeError): + # librenms_id is not convertible to int; no match will be found + existing_device = None + + if existing_device: + logger.info(f"Found existing device: {existing_device.name} (matched by librenms_id={librenms_id})") + result["existing_device"] = existing_device + result["existing_match_type"] = "librenms_id" + result["can_import"] = False + + # Check if name matches sysName + sys_name = libre_device.get("sysName") or "" + if sys_name and existing_device.name == sys_name: + result["name_matches"] = True + elif sys_name and existing_device.name != sys_name: + result["name_sync_available"] = True + result["suggested_name"] = sys_name + + # Check for serial drift on the linked device + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-": + if existing_device.serial and existing_device.serial == incoming_serial: + result["serial_confirmed"] = True + elif existing_device.serial and existing_device.serial != incoming_serial: + serial_conflict = ( + Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + ) + if serial_conflict: + result["serial_action"] = "conflict" + result["serial_duplicate"] = True + result["warnings"].append( + f"Serial conflict: incoming serial '{incoming_serial}' is already assigned to " + f"device '{serial_conflict.name}' (ID: {serial_conflict.pk}) in NetBox. " + f"Investigate which device should own this serial before updating." + ) + else: + result["serial_action"] = "update_serial" + result["warnings"].append( + f"Serial number differs (NetBox: '{existing_device.serial}', " + f"LibreNMS: '{incoming_serial}'). Hardware may have been replaced." + ) + + # Only check hostname/serial/IP if not already matched by librenms_id + if not result["existing_device"]: + # Check by hostname/name - Check both VMs and Devices for conflicts + existing_vm = VirtualMachine.objects.filter(name__iexact=hostname).first() + existing_device = Device.objects.filter(name__iexact=hostname).first() + + # If BOTH exist with same hostname, it's ambiguous - don't match either + if existing_vm and existing_device: + logger.warning( + f"Hostname conflict: Both VM '{existing_vm.name}' and Device " + f"'{existing_device.name}' exist with hostname '{hostname}'" + ) + result["warnings"].append( + f"Both a VM and Device exist with hostname '{hostname}' in NetBox. " + f"Cannot determine which to match. Please set the librenms_id custom field on the correct object." + ) + # Don't set existing_device, don't block import - let user proceed as new + # This allows them to import and then resolve the conflict manually + elif existing_vm: + logger.info(f"Found existing VM by hostname: {existing_vm.name}") + result["existing_device"] = existing_vm + result["existing_match_type"] = "hostname" + result["import_as_vm"] = True # Force VM mode since VM exists + result["warnings"].append( + f"VM with same hostname exists in NetBox as '{existing_vm.name}' (not linked to LibreNMS)" + ) + result["can_import"] = False + elif existing_device: + logger.info(f"Found existing device by hostname: {existing_device.name}") + result["existing_device"] = existing_device + result["existing_match_type"] = "hostname" + + # Check for serial conflict on hostname-matched device + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-" and existing_device.serial != incoming_serial: + serial_conflict = ( + Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + ) + if serial_conflict: + result["serial_action"] = "conflict" + result["serial_duplicate"] = True + result["warnings"].append( + f"Serial conflict: incoming serial '{incoming_serial}' is already assigned to " + f"device '{serial_conflict.name}' (ID: {serial_conflict.pk}) in NetBox. " + f"Investigate which device should own this serial before importing." + ) + else: + result["serial_action"] = "update_serial" + result["warnings"].append( + f"Hostname matches but serial differs (NetBox: '{existing_device.serial}', " + f"LibreNMS: '{incoming_serial}'). Hardware may have been replaced." + ) + else: + result["warnings"].append( + f"Device with same hostname exists in NetBox as '{existing_device.name}' (not linked to LibreNMS)" + ) + + result["can_import"] = False + + # Check by serial number (strong physical match - hardware identity) + if not result["existing_device"]: + serial = libre_device.get("serial") or "" + if serial and serial != "-" and not import_as_vm: + existing_by_serial = Device.objects.filter(serial=serial).first() + if existing_by_serial: + logger.info(f"Found existing device by serial: {existing_by_serial.name} (serial={serial})") + result["existing_device"] = existing_by_serial + result["existing_match_type"] = "serial" + result["can_import"] = False + + if existing_by_serial.name and existing_by_serial.name.lower() == hostname.lower(): + result["warnings"].append( + f"Device with same serial and hostname exists as '{existing_by_serial.name}' " + f"(not linked to LibreNMS)" + ) + result["serial_action"] = "link" + else: + result["warnings"].append( + f"Device with same serial ({serial}) exists as '{existing_by_serial.name}' " + f"but hostname differs (LibreNMS: '{hostname}'). Device may have been reinstalled." + ) + result["serial_action"] = "hostname_differs" + + # Check by primary IP (weaker match, IP could be reassigned) - only for devices + if not result["existing_device"]: + primary_ip = libre_device.get("ip") + if primary_ip and not import_as_vm: + from ipam.models import IPAddress + + existing_ip = IPAddress.objects.filter(address__net_host=primary_ip).first() + if existing_ip and existing_ip.assigned_object: + device = ( + existing_ip.assigned_object.device + if hasattr(existing_ip.assigned_object, "device") + else None + ) + if device: + result["existing_device"] = device + result["existing_match_type"] = "primary_ip" + result["warnings"].append( + f"IP address {primary_ip} already assigned to device '{device.name}' (not linked to LibreNMS)" + ) + result["can_import"] = False + + # Validate based on import type (Device or VM) + if import_as_vm: + # 2. For VMs: Validate Cluster (required) - Must be manually selected + result["cluster"]["found"] = False + result["issues"].append("Cluster must be manually selected before importing as VM") + # Provide list of available clusters for user selection (cached) + cache_key = "librenms_import_all_clusters" + all_clusters = cache.get(cache_key) + if all_clusters is None: + all_clusters = list(Cluster.objects.all()) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, all_clusters, cache_timeout) + result["cluster"]["available_clusters"] = all_clusters + + # Skip device-specific validations for VMs + result["site"]["found"] = True # Not required for VMs + result["device_type"]["found"] = True # Not required for VMs + result["device_role"]["found"] = True # Not required for VMs + + else: + # 2. For Devices: Validate Site (required) + location = libre_device.get("location", "") + site_match = find_matching_site(location) + result["site"] = site_match + + if not site_match["found"]: + result["issues"].append(f"No matching site found for location: '{location}'") + # Get alternative suggestions + if location: + all_sites = Site.objects.all()[:10] # Limit for performance + result["site"]["suggestions"] = list(all_sites) + + # 3. Validate DeviceType (required) + hardware = libre_device.get("hardware", "") + dt_match = match_librenms_hardware_to_device_type(hardware) + + result["device_type"] = dt_match + + if not dt_match["matched"]: + result["issues"].append(f"No matching device type found for hardware: '{hardware}'") + # Get some device types for user to choose from + all_device_types = DeviceType.objects.all()[:10] + result["device_type"]["suggestions"] = [ + { + "device_type": dt, + "similarity": 0.0, # No fuzzy matching, just showing options + "match_field": None, + } + for dt in all_device_types + ] + else: + # Rename 'matched' to 'found' for consistency + result["device_type"]["found"] = dt_match["matched"] + result["device_type"]["device_type"] = dt_match["device_type"] + result["device_type"]["match_type"] = dt_match["match_type"] + + # 4. DeviceRole (required) - Must be manually selected by user + logger.debug(f"[{hostname}] Issues BEFORE adding role issue: {result['issues']}") + result["device_role"]["found"] = False + result["issues"].append("Device role must be manually selected before import") + logger.debug(f"[{hostname}] Issues AFTER adding role issue: {result['issues']}") + # Provide list of available roles for user selection (cached) + cache_key = "librenms_import_all_roles" + all_roles = cache.get(cache_key) + if all_roles is None: + all_roles = list(DeviceRole.objects.all()) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, all_roles, cache_timeout) + result["device_role"]["available_roles"] = all_roles + + # 4b. Rack (optional) - Provide available racks for the matched site + if site_match["found"] and site_match["site"]: + site = site_match["site"] + # Use cache to optimize rack lookups per site + cache_key = f"librenms_import_racks_site_{site.pk}" + available_racks = cache.get(cache_key) + + if available_racks is None: + from dcim.models import Rack + from django.db.models import Q + + # Query racks for this site - include both: + # 1. Racks assigned to locations within the site + # 2. Racks directly assigned to the site (without location) + available_racks = list( + Rack.objects.filter(Q(location__site=site) | Q(site=site)) + .select_related("location", "site") + .order_by("location__name", "name") + ) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, available_racks, cache_timeout) + + result["rack"]["available_racks"] = available_racks + # Rack is optional, don't add to issues + result["rack"]["found"] = True # Mark as "found" even if None (optional field) + + # Skip VM-specific validations for devices + result["cluster"]["found"] = True # Not required for devices + + # 5. Match Platform (optional - same for both devices and VMs) + os = libre_device.get("os", "") + platform_match = find_matching_platform(os) + result["platform"] = platform_match + + if not platform_match["found"] and os: + result["warnings"].append(f"No matching platform found for OS: '{os}'") + + # 6. Additional validations + if not hostname: + result["issues"].append("Device has no hostname") + + # 7. Virtual chassis detection (only for devices, not VMs) + if include_vc_detection and not import_as_vm and api is not None: + device_id = libre_device.get("device_id") + if device_id: + try: + logger.debug(f"Calling get_virtual_chassis_data for device {device_id}") + vc_detection = get_virtual_chassis_data(api, device_id, force_refresh=force_vc_refresh) + logger.debug( + f"VC detection result: is_stack={vc_detection.get('is_stack')}, " + f"member_count={vc_detection.get('member_count')}, " + f"members={len(vc_detection.get('members', []))}" + ) + if vc_detection: + result["virtual_chassis"] = vc_detection + if vc_detection["is_stack"]: + logger.debug( + f"Virtual chassis CONFIRMED for device {hostname}: " + f"{vc_detection['member_count']} members" + ) + except Exception as e: + logger.exception(f"Exception during VC detection for device {hostname}: {e}") + result["virtual_chassis"]["detection_error"] = str(e) + else: + logger.debug(f"No device_id found for {hostname}") + + # 8. Determine if device/VM is ready to import + if result["existing_device"]: + # Already matched - can_import was already set to False + result["is_ready"] = False + # Populate role from existing device so the modal shows it + existing = result["existing_device"] + if hasattr(existing, "role") and existing.role: + result["device_role"]["found"] = True + result["device_role"]["role"] = existing.role + + # Check for device type mismatch between existing device and LibreNMS + if hasattr(existing, "device_type") and existing.device_type: + librenms_dt = result["device_type"].get("device_type") + if librenms_dt and existing.device_type.pk != librenms_dt.pk: + result["device_type_mismatch"] = True + result["warnings"].append( + f"Device type mismatch: NetBox has '{existing.device_type}' " + f"but LibreNMS reports '{librenms_dt}'. " + f"This may indicate the wrong device was matched." + ) + else: + result["can_import"] = len(result["issues"]) == 0 + + if import_as_vm: + # For VMs: only cluster is required + result["is_ready"] = result["can_import"] and result["cluster"]["found"] + else: + # For Devices: site, device_type, and device_role are required + result["is_ready"] = ( + result["can_import"] + and result["site"]["found"] + and result["device_type"]["found"] + and result["device_role"]["found"] + ) + + logger.debug( + f"Validation for {libre_device.get('hostname')} ({'VM' if import_as_vm else 'Device'}): " + f"issues={len(result['issues'])}, can_import={result['can_import']}, " + f"issues_list={result['issues']}" + ) + + return result + + except Exception as e: + logger.exception(f"Error validating device for import: {libre_device.get('hostname', 'unknown')}") + result["issues"].append(f"Validation error: {str(e)}") + return result + + +def import_single_device( + device_id: int, + server_key: str = None, + validation: dict = None, + manual_mappings: dict = None, + sync_options: dict = None, + libre_device: dict = None, +) -> dict: + """ + Import a single LibreNMS device to NetBox. + + Args: + device_id: LibreNMS device ID + server_key: LibreNMS server configuration key + validation: Pre-computed validation dict (optional) + manual_mappings: Manual object mappings (optional): + - site_id: NetBox Site ID + - device_type_id: NetBox DeviceType ID + - device_role_id: NetBox DeviceRole ID + - platform_id: NetBox Platform ID (optional) + - rack_id: NetBox Rack ID (optional) + sync_options: Sync options (optional): + - sync_interfaces: bool (default True) + - sync_cables: bool (default True) + - sync_ips: bool (default True) + - sync_fields: bool (default True) + libre_device: Pre-fetched LibreNMS device data (optional). + If provided, skips API call to fetch device info. + + Returns: + dict: Import result with structure: + { + 'success': bool, + 'device': Device object or None, + 'message': str, + 'error': str or None, + 'synced': { + 'interfaces': int, + 'cables': int, + 'ip_addresses': int + } + } + """ + try: + api = LibreNMSAPI(server_key=server_key) + + # Use pre-fetched device data if provided, otherwise fetch from API + if libre_device is None: + success, libre_device = api.get_device_info(device_id) + if not success or not libre_device: + return { + "success": False, + "device": None, + "message": "", + "error": f"Failed to retrieve device {device_id} from LibreNMS", + "synced": {}, + } + + # Validate device if validation not provided + if validation is None: + validation = validate_device_for_import(libre_device) + + # Check if device already exists + if validation.get("existing_device"): + return { + "success": False, + "device": validation["existing_device"], + "message": "", + "error": f"Device already exists: {validation['existing_device'].name}", + "synced": {}, + } + + # Use validation-derived matches, allow manual mappings to override specific fields + site = validation["site"].get("site") + device_type = validation["device_type"].get("device_type") + device_role = validation["device_role"].get("role") + platform = validation["platform"].get("platform") + rack = validation.get("rack", {}).get("rack") + + if manual_mappings: + site = Site.objects.filter(id=manual_mappings.get("site_id")).first() or site + device_type = DeviceType.objects.filter(id=manual_mappings.get("device_type_id")).first() or device_type + device_role = DeviceRole.objects.filter(id=manual_mappings.get("device_role_id")).first() or device_role + + platform_id = manual_mappings.get("platform_id") + if platform_id: + from dcim.models import Platform + + platform = Platform.objects.filter(id=platform_id).first() or platform + + rack_id = manual_mappings.get("rack_id") + if rack_id: + rack = Rack.objects.select_related("location", "site").filter(id=rack_id).first() or rack + + rack = rack or validation.get("rack", {}).get("rack") + + # Validate required fields + if not site: + return { + "success": False, + "device": None, + "message": "", + "error": "Site is required but not provided", + "synced": {}, + } + if not device_type: + return { + "success": False, + "device": None, + "message": "", + "error": "Device type is required but not provided", + "synced": {}, + } + if not device_role: + return { + "success": False, + "device": None, + "message": "", + "error": "Device role is required but not provided", + "synced": {}, + } + + # Create device in NetBox + with transaction.atomic(): + # Determine device name based on sync options + use_sysname = sync_options.get("use_sysname", True) if sync_options else True + strip_domain = sync_options.get("strip_domain", False) if sync_options else False + + device_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=strip_domain, + device_id=device_id, + ) + + # Generate import timestamp comment + import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") + + device_data = { + "name": device_name, + "site": site, + "device_type": device_type, + "role": device_role, + "status": "active" if libre_device.get("status") == 1 else "offline", + "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", + "custom_field_data": {"librenms_id": int(device_id)}, + } + + # Add optional fields + if platform: + device_data["platform"] = platform + + if rack: + device_data["rack"] = rack + + serial = libre_device.get("serial", "") + if serial and serial != "-": + device_data["serial"] = serial + + location_name = libre_device.get("location", "") + if location_name and location_name != "-": + from dcim.models import Location + + # Try to find matching location within the site + location = Location.objects.filter(site=site, name__iexact=location_name).first() + if location: + device_data["location"] = location + + # Create the device + device = Device(**device_data) + device.full_clean() + device.save() + + # Sync additional data based on options + sync_options = sync_options or {} + synced = {"interfaces": 0, "cables": 0, "ip_addresses": 0} + + try: + # Sync interfaces + if sync_options.get("sync_interfaces", True): + # This is simplified - would need proper request context + # For now, just log that it should be done + logger.info(f"Interface sync should be performed for device {device.name}") + + # Sync cables + if sync_options.get("sync_cables", True): + logger.info(f"Cable sync should be performed for device {device.name}") + + # Sync IP addresses + if sync_options.get("sync_ips", True): + logger.info(f"IP address sync should be performed for device {device.name}") + + except Exception as e: + logger.warning(f"Error during post-import sync: {str(e)}") + # Don't fail the import if sync fails + + return { + "success": True, + "device": device, + "message": f"Successfully imported device: {device.name}", + "error": None, + "synced": synced, + } + + except Exception as e: + logger.exception(f"Error importing device {device_id}") + return { + "success": False, + "device": None, + "message": "", + "error": str(e), + "synced": {}, + } + + +def get_librenms_device_by_id(api: LibreNMSAPI, device_id: int) -> dict: + """ + Retrieve a single device from LibreNMS by ID. + + Args: + api: LibreNMSAPI instance + device_id: LibreNMS device ID + + Returns: + Device dictionary or None if not found + """ + try: + # Use the dedicated API endpoint to get device by ID + success, device = api.get_device_info(device_id) + if success and device: + return device + + logger.warning(f"Device {device_id} not found in LibreNMS") + return None + except Exception as e: + logger.exception(f"Failed to get device {device_id} from LibreNMS: {e}") + return None + + +def fetch_device_with_cache( + device_id: int, + api: LibreNMSAPI, + server_key: str = None, + libre_devices_cache: dict = None, +) -> dict | None: + """ + Fetch LibreNMS device from cache or API with automatic caching. + + Checks three sources in order: + 1. Pre-fetched cache dict (if provided) + 2. Django cache (Redis/memory) + 3. LibreNMS API (caches result for future use) + + This function consolidates the device fetching pattern used throughout + the import workflow, eliminating code duplication. + + Args: + device_id: LibreNMS device ID to fetch + api: LibreNMSAPI instance for fallback API calls + server_key: Optional server key for multi-server setups (defaults to api.server_key) + libre_devices_cache: Optional pre-fetched device cache dict + + Returns: + Device dict from LibreNMS, or None if not found + + Example: + >>> # Simple usage + >>> libre_device = fetch_device_with_cache(123, api) + >>> if libre_device: + ... print(libre_device['hostname']) + >>> + >>> # With pre-fetched cache dict + >>> cache_dict = {123: {...}, 456: {...}} + >>> libre_device = fetch_device_with_cache(123, api, libre_devices_cache=cache_dict) + """ + # Check pre-fetched cache dict first (fastest) + if libre_devices_cache and device_id in libre_devices_cache: + return libre_devices_cache[device_id] + + # Check Django cache + cache_key = get_import_device_cache_key(device_id, server_key or api.server_key) + libre_device = cache.get(cache_key) + + if not libre_device: + # Fallback to API fetch + libre_device = get_librenms_device_by_id(api, device_id) + if libre_device: + # Cache for future use + cache.set(cache_key, libre_device, timeout=api.cache_timeout) + + return libre_device diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py new file mode 100644 index 0000000000..27f4449266 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -0,0 +1,253 @@ +"""Device filtering and retrieval from LibreNMS.""" + +import logging +from typing import List + +from django.core.cache import cache + +from ..librenms_api import LibreNMSAPI + +logger = logging.getLogger(__name__) + + +def get_device_count_for_filters( + api: LibreNMSAPI, + filters: dict, + clear_cache: bool = False, + show_disabled: bool = True, +) -> int: + """ + Get count of LibreNMS devices matching filters. + + This is a lightweight function to determine device count for background job + decision making. Uses the same caching as get_librenms_devices_for_import(). + + Args: + api: LibreNMS API client instance + filters: Filter dict with location, type, os, hostname, sysname keys + clear_cache: Whether to force cache refresh + show_disabled: Whether to include disabled devices + + Returns: + int: Count of devices matching filters + """ + devices = get_librenms_devices_for_import(api, filters=filters, force_refresh=clear_cache) + + # Filter out disabled devices if requested + if not show_disabled: + devices = [d for d in devices if d.get("status") == 1] + + return len(devices) + + +def get_librenms_devices_for_import( + api: LibreNMSAPI = None, + filters: dict = None, + server_key: str = None, + *, + force_refresh: bool = False, + return_cache_status: bool = False, +) -> List[dict] | tuple[List[dict], bool]: + """ + Retrieve LibreNMS devices based on filters. + + Args: + api: LibreNMSAPI instance (if not provided, creates one with server_key) + filters: Dict containing filter parameters: + - location: LibreNMS location/site filter + - type: Device type filter + - os: Operating system filter + - hostname: Hostname filter (partial match) + - sysname: System name filter (partial match) + - status: Device status filter (1=up, 0=down) + - disabled: Include disabled devices (0=active only, 1=all) + server_key: Key for specific server configuration (used if api not provided) + force_refresh: When True, bypass the cache and fetch fresh data + return_cache_status: When True, returns (devices, from_cache) tuple + + Returns: + List of device dictionaries from LibreNMS, or tuple of (devices, from_cache) + if return_cache_status is True. from_cache=True means data was loaded from + existing cache; from_cache=False means data was just fetched from LibreNMS. + """ + try: + # Use provided API instance or create a new one + if api is None: + api = LibreNMSAPI(server_key=server_key) + + # Build LibreNMS API filters using the type/query format + # LibreNMS API v0 expects ?type=X&query=Y format, not direct parameters + # NOTE: API only supports ONE type/query pair, so we'll use the most + # specific filter for the API and apply others client-side + api_filters = {} + client_filters = {} # Filters to apply after fetching from API + + if filters: + # Check for status filter first - it has special handling + if filters.get("status") is not None: + # Status filter uses special types that don't need query param + if filters["status"] == 1: + api_filters["type"] = "up" + elif filters["status"] == 0: + api_filters["type"] = "down" + + # Save ALL other filters for client-side filtering when status is used + if filters.get("location"): + client_filters["location"] = filters["location"] + if filters.get("type"): + client_filters["type"] = filters["type"] + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + else: + # Priority order for type/query filters: location > type > os > hostname > sysname + # Note: When sysname is combined with other filters, it's applied client-side for partial matching + # When sysname is alone, it uses API exact match (type=sysName) + # Note: hardware is always applied client-side for partial matching + # Use first available for API, save others for client-side filtering + if filters.get("location"): + api_filters["type"] = "location_id" + api_filters["query"] = filters["location"] + # Save remaining filters for client-side + if filters.get("type"): + client_filters["type"] = filters["type"] + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("type"): + api_filters["type"] = "type" + api_filters["query"] = filters["type"] + # Save remaining filters for client-side + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("os"): + api_filters["type"] = "os" + api_filters["query"] = filters["os"] + # Save remaining filters for client-side + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("hostname"): + api_filters["type"] = "hostname" + api_filters["query"] = filters["hostname"] + # Save sysname and hardware for client-side + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("sysname"): + # sysname-only filter: Use API exact match (type=sysName&query=) + # This is safe - returns empty if no exact match found + api_filters["type"] = "sysName" + api_filters["query"] = filters["sysname"] + # Save hardware for client-side + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("hardware"): + # hardware-only filter: apply client-side for partial matching + client_filters["hardware"] = filters["hardware"] + + # Note: disabled filter isn't directly supported by LibreNMS API + # We'll filter client-side if needed + + # Use caching to avoid repeated API calls + # Include both API and client filters in cache key + cache_key = f"librenms_devices_import_{server_key}_{hash(str(api_filters))}_{hash(str(client_filters))}" + from_cache = False + + if force_refresh: + cache.delete(cache_key) + else: + cached_result = cache.get(cache_key) + if cached_result is not None: + # No need to deepcopy - cached data isn't mutated + devices = cached_result + from_cache = True + if return_cache_status: + return devices, from_cache + return devices + + success, devices = api.list_devices(api_filters if api_filters else None) + + if not success: + logger.error(f"Failed to retrieve devices from LibreNMS: {devices}") + if return_cache_status: + return [], False + return [] + + # Apply client-side filters if any + if client_filters: + devices = _apply_client_filters(devices, client_filters) + + # Cache using configured timeout (default 300s) + # No need to deepcopy - Django's cache backend handles serialization + cache.set(cache_key, devices, timeout=api.cache_timeout) + + if return_cache_status: + return devices, from_cache + return devices + + except Exception: + logger.exception("Error retrieving LibreNMS devices for import") + if return_cache_status: + return [], False + return [] + + +def _apply_client_filters(devices: List[dict], filters: dict) -> List[dict]: + """ + Apply client-side filters to device list. + + Args: + devices: List of device dicts from LibreNMS + filters: Dict of filters to apply (location, type, os, hostname, sysname) + + Returns: + Filtered list of devices + """ + filtered = devices + + if filters.get("location"): + location_id = str(filters["location"]) + filtered = [d for d in filtered if str(d.get("location_id", "")) == location_id] + + if filters.get("type"): + device_type = filters["type"].lower() + filtered = [d for d in filtered if d.get("type", "").lower() == device_type] + + if filters.get("os"): + os_filter = filters["os"].lower() + filtered = [d for d in filtered if os_filter in d.get("os", "").lower()] + + if filters.get("hostname"): + hostname_filter = filters["hostname"].lower() + filtered = [d for d in filtered if hostname_filter in d.get("hostname", "").lower()] + + if filters.get("sysname"): + sysname_filter = filters["sysname"].lower() + filtered = [d for d in filtered if sysname_filter in d.get("sysName", "").lower()] + + if filters.get("hardware"): + hardware_filter = filters["hardware"].lower() + filtered = [d for d in filtered if hardware_filter in (d.get("hardware") or "").lower()] + + return filtered diff --git a/netbox_librenms_plugin/import_utils/permissions.py b/netbox_librenms_plugin/import_utils/permissions.py new file mode 100644 index 0000000000..9e9b4521e0 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/permissions.py @@ -0,0 +1,48 @@ +"""Permission check helpers for device import operations.""" + +import logging + +from django.core.exceptions import PermissionDenied + +logger = logging.getLogger(__name__) + + +def check_user_permissions(user, permissions): + """ + Check if user has all required permissions. + + Args: + user: The user object to check permissions for + permissions: List of permission strings (e.g., ['dcim.add_device', 'dcim.add_interface']) + + Returns: + tuple: (has_all_permissions: bool, missing_permissions: list[str]) + + Raises: + PermissionDenied: If user is None (no user context available) + """ + if user is None: + raise PermissionDenied("No user context available for permission check") + + missing = [perm for perm in permissions if not user.has_perm(perm)] + return (len(missing) == 0, missing) + + +def require_permissions(user, permissions, action_description="perform this action"): + """ + Require user has all permissions, raising PermissionDenied if not. + + Args: + user: The user object to check permissions for + permissions: List of permission strings + action_description: Human-readable description for error message + + Raises: + PermissionDenied: If user lacks any required permission + """ + has_perms, missing = check_user_permissions(user, permissions) + if not has_perms: + missing_str = ", ".join(missing) + raise PermissionDenied( + f"You do not have permission to {action_description}. Missing permissions: {missing_str}" + ) diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py new file mode 100644 index 0000000000..79db61e3fe --- /dev/null +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -0,0 +1,440 @@ +"""Virtual chassis detection, creation, and management.""" + +import logging +from typing import List + +from dcim.models import Device, VirtualChassis +from django.core.cache import cache +from django.db import transaction + +from ..librenms_api import LibreNMSAPI + +logger = logging.getLogger(__name__) + + +def empty_virtual_chassis_data() -> dict: + """Public helper for callers that need a blank VC payload.""" + + return { + "is_stack": False, + "member_count": 0, + "members": [], + "detection_error": None, + } + + +def _clone_virtual_chassis_data(data: dict | None) -> dict: + """Return a defensive copy of cached VC data to avoid shared references.""" + + if not data: + return empty_virtual_chassis_data() + + members = [] + for idx, member in enumerate(data.get("members", [])): + member_copy = member.copy() + raw_position = member_copy.get("position", idx) + try: + member_copy["position"] = int(raw_position) + except (TypeError, ValueError): + member_copy["position"] = idx + members.append(member_copy) + + member_count = data.get("member_count") or len(members) + + return { + "is_stack": bool(data.get("is_stack")), + "member_count": member_count, + "members": members, + "detection_error": data.get("detection_error"), + } + + +_VC_CACHE_VERSION = "v1" + + +def _vc_cache_key(api: LibreNMSAPI, device_id: int | str) -> str: + server_key = getattr(api, "server_key", "default") + return f"librenms_vc_detection_{_VC_CACHE_VERSION}_{server_key}_{device_id}" + + +def get_virtual_chassis_data(api: LibreNMSAPI, device_id: int | str, *, force_refresh: bool = False) -> dict: + """Fetch (and cache) virtual chassis data for a LibreNMS device.""" + + if not api or device_id is None: + return empty_virtual_chassis_data() + + cache_key = _vc_cache_key(api, device_id) + if not force_refresh: + cached = cache.get(cache_key) + if cached is not None: + return _clone_virtual_chassis_data(cached) + + detection_data = detect_virtual_chassis_from_inventory(api, device_id) + if detection_data and "detection_error" not in detection_data: + detection_data["detection_error"] = None + + cache_value = _clone_virtual_chassis_data(detection_data) if detection_data else empty_virtual_chassis_data() + + cache_timeout = getattr(api, "cache_timeout", 300) or 300 + cache.set(cache_key, cache_value, timeout=cache_timeout) + return _clone_virtual_chassis_data(cache_value) + + +def prefetch_vc_data_for_devices(api: LibreNMSAPI, device_ids: List[int], *, force_refresh: bool = False) -> None: + """ + Pre-warm the virtual chassis cache for multiple devices. + + This eliminates the 0.5-1s delay when rendering the import table + by proactively fetching VC data before validation. + + Args: + api: LibreNMSAPI instance + device_ids: List of LibreNMS device IDs to prefetch VC data for + force_refresh: When True, bypass cache and fetch fresh data + + Example: + >>> # Before rendering import table + >>> prefetch_vc_data_for_devices(api, [123, 124, 125]) + >>> # Now all validate_device_for_import() calls hit cache instantly + """ + if not api or not device_ids: + return + + logger.debug(f"Pre-warming VC cache for {len(device_ids)} devices") + + for idx, device_id in enumerate(device_ids): + # This populates the cache if empty, or skips if already cached + try: + get_virtual_chassis_data(api, device_id, force_refresh=force_refresh) + except (BrokenPipeError, ConnectionError, IOError, OSError) as e: + logger.warning(f"Connection error during VC prefetch at device {idx}: {e}") + # Stop processing if connection is broken + return + except Exception as e: + # Log but continue for other errors + logger.warning(f"Error prefetching VC data for device {device_id}: {e}") + + logger.debug(f"VC cache warming complete for {len(device_ids)} devices") + + +def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> dict: + """ + Detect if device is a stack/Virtual Chassis by analyzing ENTITY-MIB inventory. + Vendor-agnostic using standard hierarchical structure. + + Args: + api: LibreNMSAPI instance + device_id: LibreNMS device ID + + Returns: + dict with structure: + { + 'is_stack': bool, + 'member_count': int, + 'members': [ + { + 'serial': str, + 'position': int, + 'model': str, + 'name': str, + 'index': int, + 'description': str, + 'suggested_name': str # Generated using master device name + } + ] + } + Returns None if not a stack or detection fails. + + Detection Logic: + 1. Check root level (entPhysicalContainedIn=0) for parent container + 2. Find parent index (entPhysicalClass='stack' or 'chassis') + 3. Get children chassis at that parent's index + 4. If multiple chassis found -> Stack detected + """ + try: + # Get the master device info to use for naming + success, device_info = api.get_device_info(device_id) + master_name = None + if success and device_info: + master_name = device_info.get("sysName") or device_info.get("hostname") + + # Step 1: Get root level items + success, root_items = api.get_inventory_filtered(device_id, ent_physical_contained_in=0) + + if not success or not root_items: + logger.debug(f"No root inventory items found for device {device_id}") + return None + + # Step 2: Find parent container index + # Could be class="stack" or the main "chassis" + parent_index = None + for item in root_items: + item_class = item.get("entPhysicalClass") + if item_class in ["stack", "chassis"]: + parent_index = item.get("entPhysicalIndex") + logger.debug(f"VC detection: Found parent container at index {parent_index} for device {device_id}") + break + + if not parent_index: + return None + + # Step 3: Get children chassis at next level + success, child_items = api.get_inventory_filtered( + device_id, + ent_physical_class="chassis", + ent_physical_contained_in=parent_index, + ) + + if not success: + return None + + # Filter for chassis only (in case API filter didn't work) + chassis_items = [item for item in (child_items or []) if item.get("entPhysicalClass") == "chassis"] + + # Step 4: Multiple chassis = stack + if len(chassis_items) <= 1: + return None + + # Step 5: Extract member info + members = [] + for idx, chassis in enumerate(chassis_items): + raw_position = chassis.get("entPhysicalParentRelPos", idx) + try: + position = int(raw_position) + except (TypeError, ValueError): + position = idx + member_data = { + "serial": chassis.get("entPhysicalSerialNum", ""), + "position": position, + "model": chassis.get("entPhysicalModelName", ""), + "name": chassis.get("entPhysicalName", ""), + "index": chassis.get("entPhysicalIndex"), + "description": chassis.get("entPhysicalDescr", ""), + } + + # Generate suggested name if we have master name + if master_name: + member_data["suggested_name"] = _generate_vc_member_name(master_name, position + 1) + else: + member_data["suggested_name"] = f"Member-{position + 1}" + + members.append(member_data) + + # Sort by position + members.sort(key=lambda m: m["position"]) + + logger.info(f"Detected stack with {len(members)} members for device {device_id}") + + return {"is_stack": True, "member_count": len(members), "members": members} + + except Exception as e: + logger.exception(f"Error detecting virtual chassis for device {device_id}: {e}") + return None + + +def _generate_vc_member_name(master_name: str, position: int, serial: str = None) -> str: + """ + Generate name for VC member device using configured pattern from settings. + + Args: + master_name: Name of the master/primary device + position: VC position number + serial: Optional serial number of the member device + + Returns: + Generated member device name + + Examples: + pattern="-M{position}" -> "switch01-M2" + pattern=" ({position})" -> "switch01 (2)" + pattern="-SW{position}" -> "switch01-SW2" + pattern=" [{serial}]" -> "switch01 [ABC123]" + """ + # Import here to avoid circular dependency + from ..models import LibreNMSSettings + + # Get pattern from settings with fallback to default + try: + settings = LibreNMSSettings.objects.first() + pattern = settings.vc_member_name_pattern if settings else "-M{position}" + except Exception as e: + logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") + pattern = "-M{position}" + + # Prepare format variables + format_vars = { + "master_name": master_name, + "position": position, + "serial": serial or "", + } + + # Apply pattern - pattern should be suffix/prefix, not full name + try: + formatted_suffix = pattern.format(**format_vars) + return f"{master_name}{formatted_suffix}" + except KeyError as e: + logger.error(f"Invalid placeholder in VC naming pattern '{pattern}': {e}. Using default.") + return f"{master_name}-M{position}" + + +def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: + """ + Regenerate suggested VC member names using the actual master device name. + + This ensures preview shows accurate names after use_sysname and strip_domain + are applied to the master device name. + + Args: + vc_data: Virtual chassis detection data dict + master_name: The actual name that will be used for master device in NetBox + + Returns: + Updated vc_data dict with corrected suggested_name for each member + """ + if not vc_data or not vc_data.get("is_stack"): + return vc_data + + for idx, member in enumerate(vc_data.get("members", [])): + raw_position = member.get("position", idx) + try: + base_position = int(raw_position) + except (TypeError, ValueError): + base_position = idx + position = base_position + 1 # Convert to 1-based position + member["position"] = base_position + member["suggested_name"] = _generate_vc_member_name(master_name, position, serial=member.get("serial")) + + return vc_data + + +def create_virtual_chassis_with_members(master_device: Device, members_info: list, libre_device: dict): + """ + Create Virtual Chassis and member devices from detection info. + + This function creates a NetBox VirtualChassis with the master device + and all detected member devices, wrapped in a transaction for safety. + + Args: + master_device: The imported device (becomes VC master) + members_info: List of member dicts from VC detection + libre_device: Original LibreNMS device data + + Returns: + VirtualChassis: The created virtual chassis instance + + Raises: + ValidationError: If member count validation fails + IntegrityError: If duplicate serials/names are detected + Exception: For other creation errors + + Example members_info: + [ + {'serial': 'ABC123', 'position': 0, 'model': 'C9300-48U', 'name': 'Switch 1'}, + {'serial': 'ABC124', 'position': 1, 'model': 'C9300-48U', 'name': 'Switch 2'} + ] + """ + + # Store original master device state for rollback + original_master_name = master_device.name + original_vc = master_device.virtual_chassis + original_vc_position = master_device.vc_position + + try: + with transaction.atomic(): + # Rename master device to include position 1 pattern + master_device_new_name = _generate_vc_member_name(original_master_name, 1, serial=master_device.serial) + + # Check if renamed master conflicts with existing device + if Device.objects.filter(name=master_device_new_name).exclude(pk=master_device.pk).exists(): + logger.warning( + f"Cannot rename master to '{master_device_new_name}' - name already exists. " + f"Keeping original name '{original_master_name}'" + ) + master_base_name = original_master_name + else: + master_device.name = master_device_new_name + master_base_name = original_master_name + + # Create VC using original base name + vc_name = master_base_name + vc = VirtualChassis.objects.create( + name=vc_name, + master=master_device, + domain=f"librenms-{libre_device['device_id']}", + ) + + # Update master device + master_device.virtual_chassis = vc + master_device.vc_position = 1 # Master is position 1 + master_device.save() + + # Create member devices for remaining positions + position = 2 # Start at 2 (master is 1) + members_created = 0 + + for member in members_info: + # Skip if this is the master's serial + if member.get("serial") == master_device.serial: + continue + + serial = member.get("serial") + + member_rack = master_device.rack + member_location = master_device.location or ( + member_rack.location if member_rack and member_rack.location else None + ) + + # Check for duplicate serial + if serial and Device.objects.filter(serial=serial).exists(): + logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") + continue + + member_name = _generate_vc_member_name(master_base_name, position, serial=serial) + + # Check for duplicate name + if Device.objects.filter(name=member_name).exists(): + logger.warning(f"Device with name '{member_name}' already exists, skipping VC member creation") + continue + + Device.objects.create( + name=member_name, + device_type=master_device.device_type, + role=master_device.role, + site=master_device.site, + location=member_location, + rack=member_rack, + platform=master_device.platform, + serial=serial, + virtual_chassis=vc, + vc_position=position, + comments=f"VC member (LibreNMS: {member.get('name', 'Unknown')})\n" + f"Auto-created from stack inventory", + ) + members_created += 1 + position += 1 + + # Validate member count + expected_members = len([m for m in members_info if m.get("serial") != master_device.serial]) + if members_created < expected_members: + logger.warning( + f"Created {members_created} members but expected {expected_members}. " + "Some members may have been skipped due to duplicates." + ) + + logger.info( + f"Created Virtual Chassis '{vc.name}' with {vc.members.count()} total members " + f"(1 master + {members_created} additional)" + ) + + return vc + + except Exception as e: + # Rollback master device to original state + logger.error( + f"Virtual Chassis creation failed for device {master_device.name}: {e}. Rolling back master device changes." + ) + master_device.name = original_master_name + master_device.virtual_chassis = original_vc + master_device.vc_position = original_vc_position + master_device.save() + raise diff --git a/netbox_librenms_plugin/import_utils/vm_operations.py b/netbox_librenms_plugin/import_utils/vm_operations.py new file mode 100644 index 0000000000..eadd97d200 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/vm_operations.py @@ -0,0 +1,216 @@ +"""Virtual machine creation and import operations.""" + +import logging + +from dcim.models import DeviceRole +from django.utils import timezone +from virtualization.models import Cluster + +from ..librenms_api import LibreNMSAPI +from .device_operations import _determine_device_name, fetch_device_with_cache, validate_device_for_import +from .permissions import require_permissions + +logger = logging.getLogger(__name__) + + +def create_vm_from_librenms(libre_device: dict, validation: dict, use_sysname: bool = True, role=None): + """ + Create a NetBox VirtualMachine from LibreNMS device data. + + Args: + libre_device: Device data from LibreNMS + validation: Validation result from validate_device_for_import with import_as_vm=True + use_sysname: If True, prefer sysName; if False, use hostname + role: Optional DeviceRole to assign to the VM + + Returns: + Created VirtualMachine instance + + Raises: + Exception if VM cannot be created + """ + from virtualization.models import VirtualMachine + + if not validation["can_import"]: + raise ValueError(f"VM cannot be imported: {', '.join(validation['issues'])}") + + # Extract matched objects from validation + cluster = validation["cluster"]["cluster"] + platform = validation["platform"].get("platform") + + # Determine VM name - use pre-computed name if available (handles strip_domain) + vm_name = libre_device.get("_computed_name") + if not vm_name: + vm_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=False, + device_id=libre_device.get("device_id"), + ) + + # Generate import timestamp comment + import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") + + # Create the VM with librenms_id custom field + vm = VirtualMachine.objects.create( + name=vm_name, + cluster=cluster, + role=role, # Optional VM role + platform=platform, + comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", + custom_field_data={"librenms_id": int(libre_device["device_id"])}, + ) + + logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") + return vm + + +def bulk_import_vms( + vm_imports: dict[int, dict[str, int]], + api: LibreNMSAPI, + sync_options: dict = None, + libre_devices_cache: dict = None, + job=None, + user=None, +) -> dict: + """ + Import multiple LibreNMS devices as VMs in NetBox. + + Handles validation, cluster/role assignment, name determination, + and VM creation. Supports both synchronous and background job execution. + + This function consolidates VM import logic that was previously duplicated + in BulkImportDevicesView and ImportDevicesJob, ensuring consistent behavior + across synchronous and background import paths. + + Args: + vm_imports: Dict mapping device_id to {"cluster_id": int, "device_role_id": int} + api: LibreNMSAPI instance for device fetching + sync_options: Optional dict with use_sysname, strip_domain settings + libre_devices_cache: Optional pre-fetched device data cache + job: Optional JobRunner instance for background job logging/cancellation + user: User performing the import (for permission checks). If job is provided, + user is extracted from job.job.user if not explicitly passed. + + Returns: + Dict with keys: + - success: List of {"device_id": int, "device": VM, "message": str} + - failed: List of {"device_id": int, "error": str} + - skipped: List of {"device_id": int, "reason": str} + + Raises: + PermissionDenied: If user lacks required permissions + + Example: + >>> # Synchronous import from view + >>> vm_imports = {123: {"cluster_id": 5, "device_role_id": 2}} + >>> result = bulk_import_vms(vm_imports, api, sync_options, user=request.user) + >>> print(f"Created {len(result['success'])} VMs") + >>> + >>> # Background job import + >>> result = bulk_import_vms(vm_imports, api, sync_options, cache, job=self) + """ + from netbox_librenms_plugin.import_validation_helpers import ( + apply_cluster_to_validation, + apply_role_to_validation, + ) + + # Extract user from job if not explicitly provided + if user is None and job is not None: + user = getattr(job.job, "user", None) + + # Check permissions at start of bulk operation + require_permissions(user, ["virtualization.add_virtualmachine"], "import VMs") + + result = {"success": [], "failed": [], "skipped": []} + vm_ids = list(vm_imports.keys()) + + # Use job logger if available, otherwise standard logger + log = job.logger if job else logger + + for idx, vm_id in enumerate(vm_ids, start=1): + # Check for job cancellation every 5 VMs + if job and idx % 5 == 0: + job.job.refresh_from_db() + job_status = job.job.status + status_value = job_status.value if hasattr(job_status, "value") else job_status + if status_value in ("failed", "errored"): + log.warning(f"Job cancelled at VM {idx} of {len(vm_ids)}") + break + log.info(f"Imported VM {idx} of {len(vm_ids)}") + + try: + # Fetch device data (uses cache helper) + libre_device = fetch_device_with_cache(vm_id, api, api.server_key, libre_devices_cache) + + if not libre_device: + result["failed"].append( + { + "device_id": vm_id, + "error": f"Device {vm_id} not found in LibreNMS", + } + ) + log.error(f"Device {vm_id} not found in LibreNMS") + continue + + # Validate as VM + validation = validate_device_for_import(libre_device, import_as_vm=True, api=api) + + # Check if VM already exists + if validation.get("existing_device"): + result["skipped"].append( + { + "device_id": vm_id, + "reason": f"VM already exists: {validation['existing_device'].name}", + } + ) + log.info(f"VM already exists: {validation['existing_device'].name}") + continue + + # Apply manual cluster and role selections + vm_mappings = vm_imports[vm_id] + cluster_id = vm_mappings.get("cluster_id") + role_id = vm_mappings.get("device_role_id") + + if cluster_id: + cluster = Cluster.objects.filter(id=cluster_id).first() + if cluster: + apply_cluster_to_validation(validation, cluster) + + role = None + if role_id: + role = DeviceRole.objects.filter(id=role_id).first() + if role: + apply_role_to_validation(validation, role, is_vm=True) + + # Determine VM name + use_sysname = sync_options.get("use_sysname", True) if sync_options else True + strip_domain = sync_options.get("strip_domain", False) if sync_options else False + + vm_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=strip_domain, + device_id=vm_id, + ) + + # Update validation with computed name + libre_device["_computed_name"] = vm_name + + # Create VM + vm = create_vm_from_librenms(libre_device, validation, use_sysname=use_sysname, role=role) + + result["success"].append( + { + "device_id": vm_id, + "device": vm, + "message": f"VM {vm.name} created successfully", + } + ) + log.info(f"Successfully imported VM {vm.name} (ID: {vm_id})") + + except Exception as vm_error: + log.error(f"Failed to import VM {vm_id}: {vm_error}", exc_info=True) + result["failed"].append({"device_id": vm_id, "error": str(vm_error)}) + + return result From c603a295d5edf6d0864769e1392308f01661ed3d Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 26 Feb 2026 21:58:44 +0100 Subject: [PATCH 02/39] fix: update mock.patch paths for import_utils package refactor After splitting import_utils.py into a package, mock.patch decorators must target the actual submodule where each name is looked up, not the package __init__.py. Update all patch paths in test_import_utils.py and test_permissions.py to point to the correct submodules. --- .../import_utils/device_operations.py | 2 +- .../tests/test_import_utils.py | 268 +++++++++--------- .../tests/test_permissions.py | 20 +- 3 files changed, 145 insertions(+), 145 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index 539e133c0b..7d10cb959e 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -4,7 +4,7 @@ from dcim.models import Device, DeviceRole, DeviceType, Rack, Site from django.core.cache import cache -from virtualization.models import Cluster +from virtualization.models import Cluster # noqa: F401 — used by test mock.patch targets from django.db import transaction from django.utils import timezone diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 4643106c19..8e6d67c18f 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -149,8 +149,8 @@ def test_determine_device_name_fallback_to_device_id(self): class TestDeviceRetrieval: """Test device retrieval and filtering functions.""" - @patch("netbox_librenms_plugin.import_utils.cache") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.filters.cache") + @patch("netbox_librenms_plugin.import_utils.filters.LibreNMSAPI") def test_get_librenms_devices_for_import_success(self, mock_api_class, mock_cache): """Retrieve devices from LibreNMS API.""" mock_cache.get.return_value = None # Cache miss @@ -171,7 +171,7 @@ def test_get_librenms_devices_for_import_success(self, mock_api_class, mock_cach assert len(devices) == 2 assert devices[0]["hostname"] == "switch-01" - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_librenms_devices_for_import_uses_cache(self, mock_cache): """Cached results returned on repeat call.""" cached_devices = [ @@ -189,7 +189,7 @@ def test_get_librenms_devices_for_import_uses_cache(self, mock_cache): assert devices[0]["hostname"] == "cached-device" mock_api.list_devices.assert_not_called() - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_librenms_devices_for_import_cache_miss(self, mock_cache): """API called when cache empty.""" mock_cache.get.return_value = None @@ -209,7 +209,7 @@ def test_get_librenms_devices_for_import_cache_miss(self, mock_cache): mock_api.list_devices.assert_called_once() assert len(devices) == 1 - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_device_count_for_filters_success(self, mock_cache): """Returns correct count from API.""" mock_cache.get.return_value = [ @@ -225,7 +225,7 @@ def test_get_device_count_for_filters_success(self, mock_cache): assert count == 3 - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_device_count_excludes_disabled(self, mock_cache): """Count respects show_disabled filter parameter.""" mock_cache.get.return_value = [ @@ -279,7 +279,7 @@ def test_empty_virtual_chassis_data(self): assert data["members"] == [] assert data["detection_error"] is None - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.virtual_chassis.cache") def test_get_virtual_chassis_data_returns_empty_without_api(self, mock_cache): """Get VC data returns empty structure without API.""" from netbox_librenms_plugin.import_utils import get_virtual_chassis_data @@ -299,14 +299,14 @@ class TestDeviceValidation: """Test device validation for import.""" @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_site_match_found( self, mock_site_model, @@ -358,14 +358,14 @@ def test_validate_device_site_match_found( assert result["site"]["site"] == mock_site @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_site_not_found( self, mock_site_model, @@ -416,15 +416,15 @@ def test_validate_device_site_not_found( assert any("site" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") - @patch("netbox_librenms_plugin.import_utils.DeviceType") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceType") def test_validate_device_platform_match_found( self, mock_device_type, @@ -478,14 +478,14 @@ def test_validate_device_platform_match_found( assert result["platform"]["platform"] == mock_platform @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_platform_not_found( self, mock_site_model, @@ -535,14 +535,14 @@ def test_validate_device_platform_not_found( assert result["platform"]["found"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_type_match_found( self, mock_site_model, @@ -594,15 +594,15 @@ def test_validate_device_type_match_found( assert result["device_type"]["device_type"] == mock_dt @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") - @patch("netbox_librenms_plugin.import_utils.DeviceType") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceType") def test_validate_device_type_not_found( self, mock_device_type, @@ -655,14 +655,14 @@ def test_validate_device_type_not_found( assert any("device type" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_role_required( self, mock_site_model, @@ -716,14 +716,14 @@ def test_validate_device_role_required( assert any("role" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_handles_empty_location( self, mock_site_model, @@ -775,14 +775,14 @@ def test_validate_device_handles_empty_location( assert result["site"]["found"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_handles_empty_os( self, mock_site_model, @@ -833,15 +833,15 @@ def test_validate_device_handles_empty_os( assert result["platform"]["found"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") - @patch("netbox_librenms_plugin.import_utils.DeviceType") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceType") def test_validate_device_handles_empty_hardware( self, mock_device_type, @@ -894,14 +894,14 @@ def test_validate_device_handles_empty_hardware( assert result["device_type"]["matched"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_duplicate_detection( self, mock_site_model, @@ -933,14 +933,14 @@ def test_validate_device_duplicate_detection( assert result["can_import"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_returns_complete_state( self, mock_site_model, @@ -999,17 +999,17 @@ def test_validate_device_returns_complete_state( assert "cluster" in result assert "platform" in result - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.device_operations.cache") @patch("virtualization.models.Cluster") @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_import_as_vm( self, mock_site_model, @@ -1064,14 +1064,14 @@ def test_validate_device_import_as_vm( assert result["cluster"]["available_clusters"] == mock_clusters @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_existing_vm_blocks_import( self, mock_site_model, @@ -1108,14 +1108,14 @@ class TestSerialNumberMatching: """Test serial number matching in device validation.""" SERIAL_PATCHES = [ - "netbox_librenms_plugin.import_utils.Site", - "netbox_librenms_plugin.import_utils.Rack", - "netbox_librenms_plugin.import_utils.Cluster", - "netbox_librenms_plugin.import_utils.DeviceRole", - "netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type", - "netbox_librenms_plugin.import_utils.find_matching_platform", - "netbox_librenms_plugin.import_utils.find_matching_site", - "netbox_librenms_plugin.import_utils.Device", + "netbox_librenms_plugin.import_utils.device_operations.Site", + "netbox_librenms_plugin.import_utils.device_operations.Rack", + "netbox_librenms_plugin.import_utils.device_operations.Cluster", + "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", + "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", + "netbox_librenms_plugin.import_utils.device_operations.Device", "virtualization.models.VirtualMachine", ] @@ -1489,7 +1489,7 @@ def device_filter(**kwargs): self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] - with patch("netbox_librenms_plugin.import_utils.cache") as mock_cache: + with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: mock_cache.get.return_value = None from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1545,7 +1545,7 @@ def device_filter(**kwargs): self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] - with patch("netbox_librenms_plugin.import_utils.cache") as mock_cache: + with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: mock_cache.get.return_value = None from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1591,7 +1591,7 @@ def device_filter(**kwargs): self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] - with patch("netbox_librenms_plugin.import_utils.cache") as mock_cache: + with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: mock_cache.get.return_value = None from netbox_librenms_plugin.import_utils import validate_device_for_import diff --git a/netbox_librenms_plugin/tests/test_permissions.py b/netbox_librenms_plugin/tests/test_permissions.py index d366965ead..50bc2b6d04 100644 --- a/netbox_librenms_plugin/tests/test_permissions.py +++ b/netbox_librenms_plugin/tests/test_permissions.py @@ -633,8 +633,8 @@ class TestView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin): class TestBulkImportPermissions: """Tests for permission checks in bulk import functions.""" - @patch("netbox_librenms_plugin.import_utils.require_permissions") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions") + @patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") def test_bulk_import_devices_checks_permissions(self, mock_api_class, mock_require): """bulk_import_devices_shared calls require_permissions.""" from netbox_librenms_plugin.import_utils import bulk_import_devices_shared @@ -658,8 +658,8 @@ def test_bulk_import_devices_checks_permissions(self, mock_api_class, mock_requi assert "dcim.add_device" in call_args[0][1] assert "dcim.add_interface" in call_args[0][1] - @patch("netbox_librenms_plugin.import_utils.require_permissions") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions") + @patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") def test_bulk_import_devices_extracts_user_from_job(self, mock_api_class, mock_require): """bulk_import_devices_shared extracts user from job if not provided.""" from netbox_librenms_plugin.import_utils import bulk_import_devices_shared @@ -682,7 +682,7 @@ def test_bulk_import_devices_extracts_user_from_job(self, mock_api_class, mock_r call_args = mock_require.call_args assert job_user == call_args[0][0] - @patch("netbox_librenms_plugin.import_utils.require_permissions") + @patch("netbox_librenms_plugin.import_utils.vm_operations.require_permissions") def test_bulk_import_vms_checks_permissions(self, mock_require): """bulk_import_vms calls require_permissions.""" from netbox_librenms_plugin.import_utils import bulk_import_vms @@ -703,7 +703,7 @@ def test_bulk_import_vms_checks_permissions(self, mock_require): assert user == call_args[0][0] assert "virtualization.add_virtualmachine" in call_args[0][1] - @patch("netbox_librenms_plugin.import_utils.require_permissions") + @patch("netbox_librenms_plugin.import_utils.vm_operations.require_permissions") def test_bulk_import_vms_extracts_user_from_job(self, mock_require): """bulk_import_vms extracts user from job if not provided.""" from netbox_librenms_plugin.import_utils import bulk_import_vms @@ -729,7 +729,7 @@ def test_bulk_import_vms_extracts_user_from_job(self, mock_require): class TestBulkImportPermissionDenied: """Tests for permission denied behavior in bulk import.""" - @patch("netbox_librenms_plugin.import_utils.check_user_permissions") + @patch("netbox_librenms_plugin.import_utils.permissions.check_user_permissions") def test_bulk_import_devices_raises_on_missing_permissions(self, mock_check): """bulk_import_devices_shared raises PermissionDenied when permissions missing.""" import pytest @@ -748,7 +748,7 @@ def test_bulk_import_devices_raises_on_missing_permissions(self, mock_check): server_key="default", ) - @patch("netbox_librenms_plugin.import_utils.check_user_permissions") + @patch("netbox_librenms_plugin.import_utils.permissions.check_user_permissions") def test_bulk_import_vms_raises_on_missing_permissions(self, mock_check): """bulk_import_vms raises PermissionDenied when permissions missing.""" import pytest @@ -873,8 +873,8 @@ def test_htmx_rejects_external_referrer(self): class TestBulkImportVCPermission: """Tests that bulk import checks virtualchassis permission.""" - @patch("netbox_librenms_plugin.import_utils.require_permissions") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions") + @patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") def test_bulk_import_devices_checks_vc_permission(self, mock_api_class, mock_require): """bulk_import_devices_shared includes dcim.add_virtualchassis in required perms.""" from netbox_librenms_plugin.import_utils import bulk_import_devices_shared From 8e65b6d96278a15c48d777a4b6a588037ede6f4a Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 26 Feb 2026 22:10:42 +0100 Subject: [PATCH 03/39] refactor: move Cluster to module-level import in device_operations Keep Cluster as a top-level import so mock.patch targets can resolve it. Remove the redundant inline import inside validate_device_for_import. --- netbox_librenms_plugin/import_utils/device_operations.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index 7d10cb959e..c4fa96f8b5 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -4,9 +4,9 @@ from dcim.models import Device, DeviceRole, DeviceType, Rack, Site from django.core.cache import cache -from virtualization.models import Cluster # noqa: F401 — used by test mock.patch targets from django.db import transaction from django.utils import timezone +from virtualization.models import Cluster # noqa: F401 — used by test mock.patch targets from ..librenms_api import LibreNMSAPI from ..utils import ( @@ -414,7 +414,6 @@ def validate_device_for_import( # 3. Validate DeviceType (required) hardware = libre_device.get("hardware", "") dt_match = match_librenms_hardware_to_device_type(hardware) - result["device_type"] = dt_match if not dt_match["matched"]: From e92eef64a5648589d87dec9f12b026f9d4c71d9e Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 26 Feb 2026 23:17:48 +0100 Subject: [PATCH 04/39] fix: remove stale virtualization.models.Cluster patch in VM import test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that Cluster is a module-level import in device_operations, the test only needs to mock device_operations.Cluster — the extra patch on virtualization.models.Cluster was leftover from the inline-import era and caused the mock to target the wrong object. --- netbox_librenms_plugin/tests/test_import_utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 8e6d67c18f..7d209a2c79 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1000,7 +1000,6 @@ def test_validate_device_returns_complete_state( assert "platform" in result @patch("netbox_librenms_plugin.import_utils.device_operations.cache") - @patch("virtualization.models.Cluster") @patch("virtualization.models.VirtualMachine") @patch("netbox_librenms_plugin.import_utils.device_operations.Device") @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") @@ -1014,14 +1013,13 @@ def test_validate_device_import_as_vm( self, mock_site_model, mock_rack, - mock_cluster_module, + mock_cluster, mock_role, mock_match_type, mock_find_platform, mock_find_site, mock_device, mock_vm, - mock_cluster_local, mock_cache, ): """Import as VM mode uses cluster instead of site/device_type.""" @@ -1045,8 +1043,7 @@ def test_validate_device_import_as_vm( } mock_role.objects.all.return_value = [] mock_clusters = [MagicMock(id=1, name="VMware Cluster")] - # Cluster is imported locally in the VM path, so we need to mock it there - mock_cluster_local.objects.all.return_value = mock_clusters + mock_cluster.objects.all.return_value = mock_clusters mock_cache.get.return_value = None # Force cache miss to trigger Cluster.objects.all() mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] From d25344bdce71251527f06617cabcb5796d78ed31 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Fri, 27 Feb 2026 15:49:20 +0100 Subject: [PATCH 05/39] feat: honour use_sysname and strip_domain in validation path (port from #227) - Add use_sysname/strip_domain params to validate_device_for_import - Store resolved name in validation result as resolved_name - Update import_single_device, bulk_import_devices_shared, bulk_import_vms to extract and pass naming prefs from sync_options - Add _resolve_naming_preferences() helper in actions.py with fallback chain: POST data -> user pref -> LibreNMSSettings -> plugin default - Update DeviceImportHelperMixin and BulkImportConfirmView to pass prefs - Replace _determine_device_name() in BulkImportConfirmView with resolved_name - Add TestDeviceNamingPreferences (5 tests) to test_import_utils.py --- .../import_utils/bulk_import.py | 9 +- .../import_utils/device_operations.py | 21 ++- .../import_utils/vm_operations.py | 10 +- .../tests/test_import_utils.py | 164 ++++++++++++++++++ .../views/imports/actions.py | 51 +++++- 5 files changed, 243 insertions(+), 12 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 8f55e0b6cf..c6ef9090d3 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -122,7 +122,14 @@ def bulk_import_devices_shared( logger.error(error_msg) continue - validation = validate_device_for_import(libre_device, api=api) + use_sysname_opt = sync_options.get("use_sysname", True) if sync_options else True + strip_domain_opt = sync_options.get("strip_domain", False) if sync_options else False + validation = validate_device_for_import( + libre_device, + api=api, + use_sysname=use_sysname_opt, + strip_domain=strip_domain_opt, + ) # Build manual mappings from validation + any provided overrides device_mappings = {} diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index c4fa96f8b5..0ee983a20a 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -83,6 +83,8 @@ def validate_device_for_import( *, include_vc_detection: bool = True, force_vc_refresh: bool = False, + use_sysname: bool = True, + strip_domain: bool = False, ) -> dict: """ Validate if a LibreNMS device can be imported to NetBox. @@ -101,6 +103,8 @@ def validate_device_for_import( api: Optional LibreNMSAPI instance for virtual chassis detection include_vc_detection: Skip VC detection when False to speed up bulk operations force_vc_refresh: When True, bypass cached VC data and re-query LibreNMS + use_sysname: If True, prefer sysName over hostname (matches import behaviour) + strip_domain: If True, strip domain suffix from device name Returns: dict: Validation result with structure: @@ -149,6 +153,7 @@ def validate_device_for_import( "is_ready": False, "can_import": False, "import_as_vm": import_as_vm, + "resolved_name": None, # Final device name after applying user preferences "existing_device": None, "existing_match_type": None, # Track how existing device was matched "serial_action": None, # None, "link", "conflict", "update_serial", "hostname_differs" @@ -195,7 +200,13 @@ def validate_device_for_import( # 1. Check if device/VM already exists in NetBox # Always check both Devices AND VMs to properly detect existing objects librenms_id = libre_device.get("device_id") - hostname = libre_device.get("hostname", "") + hostname = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=strip_domain, + device_id=librenms_id, + ) + result["resolved_name"] = hostname logger.debug( f"Checking for existing device/VM: " f"librenms_id={librenms_id} (type={type(librenms_id).__name__}), " @@ -625,7 +636,13 @@ def import_single_device( # Validate device if validation not provided if validation is None: - validation = validate_device_for_import(libre_device) + use_sysname_opt = sync_options.get("use_sysname", True) if sync_options else True + strip_domain_opt = sync_options.get("strip_domain", False) if sync_options else False + validation = validate_device_for_import( + libre_device, + use_sysname=use_sysname_opt, + strip_domain=strip_domain_opt, + ) # Check if device already exists if validation.get("existing_device"): diff --git a/netbox_librenms_plugin/import_utils/vm_operations.py b/netbox_librenms_plugin/import_utils/vm_operations.py index eadd97d200..737b0d7b2b 100644 --- a/netbox_librenms_plugin/import_utils/vm_operations.py +++ b/netbox_librenms_plugin/import_utils/vm_operations.py @@ -154,7 +154,15 @@ def bulk_import_vms( continue # Validate as VM - validation = validate_device_for_import(libre_device, import_as_vm=True, api=api) + use_sysname_opt = sync_options.get("use_sysname", True) if sync_options else True + strip_domain_opt = sync_options.get("strip_domain", False) if sync_options else False + validation = validate_device_for_import( + libre_device, + import_as_vm=True, + api=api, + use_sysname=use_sysname_opt, + strip_domain=strip_domain_opt, + ) # Check if VM already exists if validation.get("existing_device"): diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 7d209a2c79..9f48fc3fdc 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -2172,3 +2172,167 @@ def test_platform_out_of_sync(self): assert result["platform_synced"] is False assert result["all_synced"] is False + + +class TestDeviceNamingPreferences: + """Test that validation honours use_sysname and strip_domain user preferences.""" + + def _setup_no_existing(self, mocks): + """Configure mocks so no existing device is found.""" + mock_vm = mocks[-1] # VirtualMachine + mock_device = mocks[-2] # Device + mock_find_site = mocks[-3] + mock_find_platform = mocks[-4] + mock_match_type = mocks[-5] + mock_role = mocks[-6] + mock_rack = mocks[-8] + mock_site_model = mocks[-9] + + mock_vm.objects.filter.return_value.first.return_value = None + mock_device.objects.filter.return_value.first.return_value = None + mock_find_site.return_value = { + "found": False, + "site": None, + "match_type": None, + "confidence": 0.0, + } + mock_find_platform.return_value = { + "found": False, + "platform": None, + "match_type": None, + } + mock_match_type.return_value = { + "matched": False, + "device_type": None, + "match_type": None, + } + mock_role.objects.all.return_value = [] + mock_rack.objects.filter.return_value = [] + mock_site_model.objects.all.return_value = [] + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_resolved_name_uses_sysname_by_default(self, *mocks): + """Default use_sysname=True uses sysName for resolved_name.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "10.0.0.1", + "sysName": "core-switch", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + assert result["resolved_name"] == "core-switch" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_resolved_name_uses_hostname_when_sysname_disabled(self, *mocks): + """use_sysname=False uses hostname for resolved_name.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "10.0.0.1", + "sysName": "core-switch", + } + result = validate_device_for_import( + device_data, + include_vc_detection=False, + use_sysname=False, + ) + assert result["resolved_name"] == "10.0.0.1" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_resolved_name_strips_domain(self, *mocks): + """strip_domain=True strips the domain suffix.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "switch-01.example.com", + "sysName": "switch-01.example.com", + } + result = validate_device_for_import( + device_data, + include_vc_detection=False, + strip_domain=True, + ) + assert result["resolved_name"] == "switch-01" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_duplicate_detection_uses_resolved_name(self, *mocks): + """Duplicate detection should match against the resolved name, not raw hostname.""" + self._setup_no_existing(mocks) + + mock_device = mocks[-2] # Device + existing = MagicMock() + existing.name = "core-switch" + existing.serial = "" + mock_device.objects.filter.return_value.first.side_effect = [None, existing] + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 999, + "hostname": "10.0.0.1", + "sysName": "core-switch", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_device"] == existing + assert result["existing_match_type"] == "hostname" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_backward_compatible_defaults(self, *mocks): + """Calling without naming params produces resolved_name in result.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "switch-01", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + assert "resolved_name" in result + assert result["resolved_name"] == "switch-01" diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 5e8a9bc4c5..4bed12e4ae 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -29,12 +29,41 @@ fetch_model_by_id, ) from netbox_librenms_plugin.tables.device_status import DeviceImportTable -from netbox_librenms_plugin.utils import save_user_pref +from netbox_librenms_plugin.utils import get_user_pref, save_user_pref from netbox_librenms_plugin.views.mixins import LibreNMSAPIMixin, LibreNMSPermissionMixin logger = logging.getLogger(__name__) +def _resolve_naming_preferences(request) -> tuple[bool, bool]: + """Resolve use_sysname/strip_domain: POST data → user pref → plugin settings.""" + if "use-sysname-toggle" in request.POST: + use_sysname = request.POST.get("use-sysname-toggle") == "on" + else: + pref = get_user_pref(request, "plugins.netbox_librenms_plugin.use_sysname") + if pref is not None: + use_sysname = pref + else: + from netbox_librenms_plugin.models import LibreNMSSettings + + settings = LibreNMSSettings.objects.first() + use_sysname = getattr(settings, "use_sysname_default", True) if settings else True + + if "strip-domain-toggle" in request.POST: + strip_domain = request.POST.get("strip-domain-toggle") == "on" + else: + pref = get_user_pref(request, "plugins.netbox_librenms_plugin.strip_domain") + if pref is not None: + strip_domain = pref + else: + from netbox_librenms_plugin.models import LibreNMSSettings + + settings = LibreNMSSettings.objects.first() + strip_domain = getattr(settings, "strip_domain_default", False) if settings else False + + return use_sysname, strip_domain + + class DeviceImportHelperMixin: """Mixin providing common validation and rendering helpers for device import views.""" @@ -113,11 +142,16 @@ def get_validated_device_with_selections(self, device_id: int, request) -> tuple # This checks: user preference, cache status, and VM vs Device enable_vc = not is_vm and self._should_enable_vc_detection(device_id, request) + # Extract naming preferences: POST data (hx-include) → user pref → plugin settings. + use_sysname, strip_domain = _resolve_naming_preferences(request) + validation = validate_device_for_import( libre_device, import_as_vm=is_vm, api=self.librenms_api if enable_vc else None, include_vc_detection=enable_vc, + use_sysname=use_sysname, + strip_domain=strip_domain, ) validation["import_as_vm"] = is_vm @@ -259,19 +293,20 @@ def post(self, request): rack_id = selections["rack_id"] is_vm = bool(cluster_id) - validation = validate_device_for_import(libre_device, import_as_vm=is_vm, api=self.librenms_api) + validation = validate_device_for_import( + libre_device, + import_as_vm=is_vm, + api=self.librenms_api, + use_sysname=use_sysname, + strip_domain=strip_domain, + ) # Mark validation with VC detection flag for proper URL generation in table # Bulk confirm should respect the initial filter's VC detection preference vc_requested = request.GET.get("enable_vc_detection") == "true" validation["_vc_detection_enabled"] = vc_requested - device_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=strip_domain, - device_id=device_id, - ) + device_name = validation["resolved_name"] if validation.get("virtual_chassis", {}).get("is_stack") and device_name: validation["virtual_chassis"] = update_vc_member_suggested_names( From d977fe3608d225086277238f7940bcaf7c426abe Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 1 Mar 2026 15:34:46 +0100 Subject: [PATCH 06/39] feat: port serial-matching and conflict-resolution fixes to split import_utils package Port 4 commits from feat/serial-matching-and-conflict-resolution: - 097ea42: Fix validation readiness (VMs require cluster.found), hostname match always sets has_actions, guard platform sync forms for Device only, use _resolve_naming_preferences in BulkImportConfirmView, add else branch for unmatched hardware in _build_sync_info, reuse resolved_name in DeviceConflictActionView actions, fix test patch targets - c0d7bfc: Block import when issues present (can_import/is_ready require no issues), modal title uses resolved_name, dedup LibreNMSSettings DB query in _resolve_naming_preferences, guard hostname form for VMs - 13bafe1: Compare existing device/VM name against resolved_name instead of raw sysName, scope mismatch force gate to link/update/update_serial/ update_type only, add librenms_id collision check before linking - dad0d22: Update device_role dict in-place during refresh to preserve schema keys (available_roles etc.) --- .../import_utils/bulk_import.py | 13 ++-- .../import_utils/device_operations.py | 14 ++-- .../tables/device_status.py | 2 +- .../htmx/device_validation_details.html | 11 ++- .../tests/test_import_utils.py | 30 +++++++- .../views/imports/actions.py | 74 ++++++++++++++----- 6 files changed, 106 insertions(+), 38 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index c6ef9090d3..120c7bda64 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -283,22 +283,25 @@ def _refresh_existing_device(validation: dict) -> None: if refreshed: validation["existing_device"] = refreshed if hasattr(refreshed, "role") and refreshed.role: - validation["device_role"] = {"found": True, "role": refreshed.role} + validation["device_role"]["found"] = True + validation["device_role"]["role"] = refreshed.role else: # Device was deleted since caching — recompute readiness validation["existing_device"] = None validation["existing_match_type"] = None - validation["can_import"] = True if validation.get("import_as_vm"): - validation["is_ready"] = bool( - validation.get("site", {}).get("found") and validation.get("device_role", {}).get("found") + required_found = ( + validation.get("site", {}).get("found") + and validation.get("cluster", {}).get("found") + and validation.get("device_role", {}).get("found") ) else: - validation["is_ready"] = bool( + required_found = ( validation.get("site", {}).get("found") and validation.get("device_type", {}).get("found") and validation.get("device_role", {}).get("found") ) + validation["can_import"] = validation["is_ready"] = bool(required_found and not validation.get("issues")) except Exception as e: existing_id = getattr(existing, "pk", "unknown") if existing else "none" logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index 0ee983a20a..eeb495b6a3 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -230,12 +230,11 @@ def validate_device_for_import( result["import_as_vm"] = True # Force VM mode since VM exists result["can_import"] = False - # Check if name matches sysName + # Check if name matches resolved name (accounts for use_sysname/strip_domain) # Note: name_sync_available/suggested_name are intentionally not set for VMs # because UpdateDeviceNameView only supports Device objects; VM name-sync # would require a separate implementation. - sys_name = libre_device.get("sysName") or "" - if sys_name and existing_vm.name == sys_name: + if hostname and existing_vm.name == hostname: result["name_matches"] = True # Check for existing Device (by librenms_id custom field) @@ -253,13 +252,12 @@ def validate_device_for_import( result["existing_match_type"] = "librenms_id" result["can_import"] = False - # Check if name matches sysName - sys_name = libre_device.get("sysName") or "" - if sys_name and existing_device.name == sys_name: + # Check if name matches resolved name (accounts for use_sysname/strip_domain) + if hostname and existing_device.name == hostname: result["name_matches"] = True - elif sys_name and existing_device.name != sys_name: + elif hostname and existing_device.name != hostname: result["name_sync_available"] = True - result["suggested_name"] = sys_name + result["suggested_name"] = hostname # Check for serial drift on the linked device incoming_serial = libre_device.get("serial") or "" diff --git a/netbox_librenms_plugin/tables/device_status.py b/netbox_librenms_plugin/tables/device_status.py index 71149e58fb..3a548bcd96 100644 --- a/netbox_librenms_plugin/tables/device_status.py +++ b/netbox_librenms_plugin/tables/device_status.py @@ -463,7 +463,7 @@ def render_actions(self, value, record): match_type = validation.get("existing_match_type", "") serial_action = validation.get("serial_action") has_mismatch = validation.get("device_type_mismatch", False) - has_actions = match_type in ("hostname", "serial") and serial_action is not None + has_actions = match_type == "hostname" or (match_type == "serial" and serial_action is not None) has_name_sync = validation.get("name_sync_available", False) has_sync_needed = match_type == "librenms_id" and serial_action in ("update_serial", "conflict") diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html index 974e8b79ec..2bb7fb7f8c 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html @@ -18,7 +18,7 @@
{% else %} {% endif %} - Import Validation: {{ libre_device.sysName|default:libre_device.hostname }} + Import Validation: {{ validation.resolved_name|default:libre_device.sysName|default:libre_device.hostname }}
@@ -256,6 +256,7 @@
Device Information
{{ validation.existing_device.platform }} {% if sync_info and not sync_info.platform_synced %} {% if sync_info.platform_info.platform_exists %} + {% if not validation.import_as_vm and not validation.existing_device.cluster %}
@@ -266,6 +267,7 @@
Device Information
+ {% endif %} {% else %} @@ -278,7 +280,7 @@
Device Information
{{ validation.platform.platform.name }} {% elif sync_info and sync_info.platform_info.platform_exists %} Not set - {% if validation.existing_device %} + {% if validation.existing_device and not validation.import_as_vm and not validation.existing_device.cluster %}
@@ -401,6 +403,11 @@
Device Information
Import blocked: The incoming serial number is already assigned to another device in NetBox. Resolve the duplicate serial before linking. + {% elif validation.import_as_vm %} +
+ + Hostname match found for a VM — use the import action to proceed. +
{% else %}
tuple[bool, bool]: """Resolve use_sysname/strip_domain: POST data → user pref → plugin settings.""" + from netbox_librenms_plugin.models import LibreNMSSettings + + settings = None + if "use-sysname-toggle" in request.POST: use_sysname = request.POST.get("use-sysname-toggle") == "on" else: @@ -44,8 +48,6 @@ def _resolve_naming_preferences(request) -> tuple[bool, bool]: if pref is not None: use_sysname = pref else: - from netbox_librenms_plugin.models import LibreNMSSettings - settings = LibreNMSSettings.objects.first() use_sysname = getattr(settings, "use_sysname_default", True) if settings else True @@ -56,9 +58,8 @@ def _resolve_naming_preferences(request) -> tuple[bool, bool]: if pref is not None: strip_domain = pref else: - from netbox_librenms_plugin.models import LibreNMSSettings - - settings = LibreNMSSettings.objects.first() + if settings is None: + settings = LibreNMSSettings.objects.first() strip_domain = getattr(settings, "strip_domain_default", False) if settings else False return use_sysname, strip_domain @@ -257,8 +258,7 @@ def post(self, request): status=400, ) - use_sysname = request.POST.get("use-sysname-toggle") == "on" - strip_domain = request.POST.get("strip-domain-toggle") == "on" + use_sysname, strip_domain = _resolve_naming_preferences(request) devices = [] errors = [] @@ -795,6 +795,8 @@ def _build_sync_info(libre_device, existing_device): librenms_device_type = hw_match["device_type"] if not existing_device.device_type or existing_device.device_type.pk != librenms_device_type.pk: device_type_synced = False + else: + device_type_synced = False all_synced = serial_synced and platform_synced and device_type_synced @@ -874,9 +876,10 @@ def post(self, request, device_id): if not libre_device: return HttpResponse("LibreNMS device not found", status=404) - # Require force flag when device type mismatches + # Require force flag when device type mismatches, but only for actions that use it + _FORCE_REQUIRED_ACTIONS = {"link", "update", "update_serial", "update_type"} force = request.POST.get("force") == "on" - if validation.get("device_type_mismatch") and not force: + if validation.get("device_type_mismatch") and action in _FORCE_REQUIRED_ACTIONS and not force: return HttpResponse( "Device type mismatch detected. Check the force checkbox to proceed.", status=400, @@ -889,11 +892,32 @@ def post(self, request, device_id): librenms_id = libre_device.get("device_id") + # Check for LibreNMS ID collision before any linking action + if action in {"link", "update", "update_serial"}: + id_conflict = ( + Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)) + .exclude(pk=existing_device.pk) + .first() + ) + if id_conflict: + return HttpResponse( + f"LibreNMS ID conflict: ID {librenms_id} is already assigned to device " + f"'{id_conflict.name}' (ID: {id_conflict.pk})", + status=409, + ) + if action == "link": # Link to LibreNMS and update name from LibreNMS data - use_sysname = request.POST.get("use-sysname-toggle") == "on" - strip_domain = request.POST.get("strip-domain-toggle") == "on" - hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) + resolved_name = validation.get("resolved_name") + hostname = ( + resolved_name + if resolved_name + else _determine_device_name( + libre_device, + use_sysname=request.POST.get("use-sysname-toggle") == "on", + strip_domain=request.POST.get("strip-domain-toggle") == "on", + ) + ) existing_device.custom_field_data["librenms_id"] = int(librenms_id) existing_device.name = hostname if librenms_device_type: @@ -903,10 +927,17 @@ def post(self, request, device_id): elif action == "update": # Update hostname, serial, and link to LibreNMS - use_sysname = request.POST.get("use-sysname-toggle") == "on" - strip_domain = request.POST.get("strip-domain-toggle") == "on" + resolved_name = validation.get("resolved_name") incoming_serial = libre_device.get("serial") or "" - hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) + hostname = ( + resolved_name + if resolved_name + else _determine_device_name( + libre_device, + use_sysname=request.POST.get("use-sysname-toggle") == "on", + strip_domain=request.POST.get("strip-domain-toggle") == "on", + ) + ) existing_device.custom_field_data["librenms_id"] = int(librenms_id) if incoming_serial and incoming_serial != "-": conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() @@ -949,9 +980,16 @@ def post(self, request, device_id): elif action == "sync_name": # Sync device name from LibreNMS (e.g., IP → sysName) - use_sysname = request.POST.get("use-sysname-toggle") == "on" - strip_domain = request.POST.get("strip-domain-toggle") == "on" - hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) + resolved_name = validation.get("resolved_name") + hostname = ( + resolved_name + if resolved_name + else _determine_device_name( + libre_device, + use_sysname=request.POST.get("use-sysname-toggle") == "on", + strip_domain=request.POST.get("strip-domain-toggle") == "on", + ) + ) existing_device.name = hostname existing_device.save() logger.info(f"Synced name on device '{existing_device.name}' from LibreNMS") From 46d6b084de4c3289cd2ade2c47fe9a34d7c687d2 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 1 Mar 2026 15:43:57 +0100 Subject: [PATCH 07/39] fix: add librenms_id collision mock to force-with-mismatch tests The librenms_id conflict check added in the port requires Device.objects.filter().exclude().first() to be mocked for all tests using link/update/update_serial actions. --- netbox_librenms_plugin/tests/test_import_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 4310df3373..96c2016037 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1906,6 +1906,7 @@ def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cach patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1950,6 +1951,7 @@ def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cach patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() From 7408b7ad45a2eb9d4e7a7cb4c3e5717cffe74c7a Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sat, 28 Feb 2026 22:09:18 +0100 Subject: [PATCH 08/39] feat: librenms_id refactor from int to JSON --- .../import_utils/bulk_import.py | 2 + .../import_utils/device_operations.py | 14 +++-- .../import_utils/vm_operations.py | 11 +++- netbox_librenms_plugin/librenms_api.py | 8 ++- netbox_librenms_plugin/tables/interfaces.py | 6 +- .../tests/test_import_utils.py | 16 ++--- netbox_librenms_plugin/utils.py | 60 +++++++++++++++++++ .../views/base/cables_view.py | 20 +++++-- .../views/base/ip_addresses_view.py | 7 ++- .../views/imports/actions.py | 11 ++-- .../views/object_sync/devices.py | 13 +++- .../views/object_sync/vms.py | 4 +- .../views/sync/interfaces.py | 4 +- 13 files changed, 138 insertions(+), 38 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 120c7bda64..0409b35da3 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -129,6 +129,7 @@ def bulk_import_devices_shared( api=api, use_sysname=use_sysname_opt, strip_domain=strip_domain_opt, + server_key=api.server_key, ) # Build manual mappings from validation + any provided overrides @@ -487,6 +488,7 @@ def process_device_filters( api=api_for_validation, include_vc_detection=vc_detection_enabled, force_vc_refresh=clear_cache, + server_key=api.server_key, ) except (BrokenPipeError, ConnectionError, IOError) as e: if request: diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index eeb495b6a3..c21d0bfb7a 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -85,6 +85,7 @@ def validate_device_for_import( force_vc_refresh: bool = False, use_sysname: bool = True, strip_domain: bool = False, + server_key: str = "default", ) -> dict: """ Validate if a LibreNMS device can be imported to NetBox. @@ -216,9 +217,10 @@ def validate_device_for_import( from virtualization.models import VirtualMachine # Check for existing VM first (by librenms_id custom field) - # Always query with int to match custom field type try: - existing_vm = VirtualMachine.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() + from netbox_librenms_plugin.utils import find_by_librenms_id + + existing_vm = find_by_librenms_id(VirtualMachine, int(librenms_id), server_key) except (ValueError, TypeError): # librenms_id is not convertible to int; no match will be found existing_vm = None @@ -238,10 +240,11 @@ def validate_device_for_import( result["name_matches"] = True # Check for existing Device (by librenms_id custom field) - # Always query with int to match custom field type if not result["existing_device"]: try: - existing_device = Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() + from netbox_librenms_plugin.utils import find_by_librenms_id + + existing_device = find_by_librenms_id(Device, int(librenms_id), server_key) except (ValueError, TypeError): # librenms_id is not convertible to int; no match will be found existing_device = None @@ -640,6 +643,7 @@ def import_single_device( libre_device, use_sysname=use_sysname_opt, strip_domain=strip_domain_opt, + server_key=server_key or "default", ) # Check if device already exists @@ -725,7 +729,7 @@ def import_single_device( "role": device_role, "status": "active" if libre_device.get("status") == 1 else "offline", "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - "custom_field_data": {"librenms_id": int(device_id)}, + "custom_field_data": {"librenms_id": {(server_key or "default"): int(device_id)}}, } # Add optional fields diff --git a/netbox_librenms_plugin/import_utils/vm_operations.py b/netbox_librenms_plugin/import_utils/vm_operations.py index 737b0d7b2b..1272871a5a 100644 --- a/netbox_librenms_plugin/import_utils/vm_operations.py +++ b/netbox_librenms_plugin/import_utils/vm_operations.py @@ -13,7 +13,9 @@ logger = logging.getLogger(__name__) -def create_vm_from_librenms(libre_device: dict, validation: dict, use_sysname: bool = True, role=None): +def create_vm_from_librenms( + libre_device: dict, validation: dict, use_sysname: bool = True, role=None, server_key: str = "default" +): """ Create a NetBox VirtualMachine from LibreNMS device data. @@ -58,7 +60,7 @@ def create_vm_from_librenms(libre_device: dict, validation: dict, use_sysname: b role=role, # Optional VM role platform=platform, comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - custom_field_data={"librenms_id": int(libre_device["device_id"])}, + custom_field_data={"librenms_id": {server_key: int(libre_device["device_id"])}}, ) logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") @@ -162,6 +164,7 @@ def bulk_import_vms( api=api, use_sysname=use_sysname_opt, strip_domain=strip_domain_opt, + server_key=api.server_key, ) # Check if VM already exists @@ -206,7 +209,9 @@ def bulk_import_vms( libre_device["_computed_name"] = vm_name # Create VM - vm = create_vm_from_librenms(libre_device, validation, use_sysname=use_sysname, role=role) + vm = create_vm_from_librenms( + libre_device, validation, use_sysname=use_sysname, role=role, server_key=api.server_key + ) result["success"].append( { diff --git a/netbox_librenms_plugin/librenms_api.py b/netbox_librenms_plugin/librenms_api.py index 5de9db6c25..8e2b2b3ac0 100644 --- a/netbox_librenms_plugin/librenms_api.py +++ b/netbox_librenms_plugin/librenms_api.py @@ -190,7 +190,9 @@ def get_librenms_id(self, obj): If found via API, stores ID in custom field if available, otherwise caches the value. """ - librenms_id = obj.cf.get("librenms_id") + from netbox_librenms_plugin.utils import get_librenms_device_id + + librenms_id = get_librenms_device_id(obj, self.server_key) if librenms_id: return librenms_id @@ -254,7 +256,9 @@ def _store_librenms_id(self, obj, librenms_id): None """ if "librenms_id" in obj.cf: - obj.custom_field_data["librenms_id"] = librenms_id + from netbox_librenms_plugin.utils import set_librenms_device_id + + set_librenms_device_id(obj, librenms_id, self.server_key) obj.save() else: # Use cache as fallback diff --git a/netbox_librenms_plugin/tables/interfaces.py b/netbox_librenms_plugin/tables/interfaces.py index 5ceb30bf51..1c5d62b1ec 100644 --- a/netbox_librenms_plugin/tables/interfaces.py +++ b/netbox_librenms_plugin/tables/interfaces.py @@ -13,6 +13,7 @@ convert_speed_to_kbps, format_mac_address, get_interface_name_field, + get_librenms_device_id, get_missing_vlan_warning, get_table_paginate_count, get_tagged_vlan_css_class, @@ -46,11 +47,12 @@ class Meta: "id": "librenms-interface-table", } - def __init__(self, *args, device=None, interface_name_field=None, vlan_groups=None, **kwargs): + def __init__(self, *args, device=None, interface_name_field=None, vlan_groups=None, server_key="default", **kwargs): """Initialize table with device context and interface name field.""" self.device = device self.interface_name_field = interface_name_field or get_interface_name_field() self.vlan_groups = vlan_groups or [] + self.server_key = server_key # Update column accessors after initialization for column in ["selection", "name"]: @@ -360,7 +362,7 @@ def render_librenms_id(self, value, record): if not netbox_interface: return mark_safe(f'{value}') - netbox_librenms_id = netbox_interface.custom_field_data.get("librenms_id") + netbox_librenms_id = get_librenms_device_id(netbox_interface, self.server_key) if netbox_librenms_id is None: return mark_safe( diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 96c2016037..a537f95d1c 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1346,7 +1346,7 @@ def test_librenms_id_match_shows_serial_confirmed(self): def device_filter(**kwargs): result = MagicMock() - if "custom_field_data__librenms_id" in kwargs: + if any(k.startswith("custom_field_data__librenms_id") for k in kwargs): result.first.return_value = existing else: result.first.return_value = None @@ -1386,7 +1386,7 @@ def test_librenms_id_match_detects_serial_drift(self): def device_filter(**kwargs): result = MagicMock() - if "custom_field_data__librenms_id" in kwargs: + if any(k.startswith("custom_field_data__librenms_id") for k in kwargs): result.first.return_value = existing elif "serial" in kwargs: result.first.return_value = None @@ -1428,7 +1428,7 @@ def test_librenms_id_match_still_validates_site(self): def device_filter(**kwargs): result = MagicMock() - if "custom_field_data__librenms_id" in kwargs: + if any(k.startswith("custom_field_data__librenms_id") for k in kwargs): result.first.return_value = existing else: result.first.return_value = None @@ -1665,7 +1665,7 @@ def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache) view.post(request, device_id=10) - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} assert existing_device.name == "switch-01.example.com" existing_device.save.assert_called_once() @@ -1705,7 +1705,7 @@ def test_update_action_sets_hostname_serial_and_librenms_id(self, mock_cache_key view.post(request, device_id=10) - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} assert existing_device.serial == "NEW-SERIAL" assert existing_device.name == "new-name.example.com" existing_device.save.assert_called_once() @@ -1741,7 +1741,7 @@ def test_update_serial_action_updates_serial_only(self, mock_cache_key, mock_cac view.post(request, device_id=10) - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} assert existing_device.serial == "NEW-SERIAL" # Name should NOT be changed by update_serial assert existing_device.name == "switch-01" @@ -1912,7 +1912,7 @@ def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cach view.post(request, device_id=10) - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} existing_device.save.assert_called_once() @patch("netbox_librenms_plugin.views.imports.actions.cache") @@ -1958,7 +1958,7 @@ def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cach view.post(request, device_id=10) assert existing_device.device_type == librenms_device_type - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} existing_device.save.assert_called_once() @patch("netbox_librenms_plugin.views.imports.actions.cache") diff --git a/netbox_librenms_plugin/utils.py b/netbox_librenms_plugin/utils.py index 4a5bf113a4..aae146b187 100644 --- a/netbox_librenms_plugin/utils.py +++ b/netbox_librenms_plugin/utils.py @@ -9,6 +9,66 @@ from utilities.paginator import get_paginate_count as netbox_get_paginate_count +def get_librenms_device_id(obj, server_key: str = "default"): + """ + Get the LibreNMS device/port ID for a specific server from the JSON custom field. + + Supports both the legacy integer format and the new multi-server JSON format:: + + Legacy: librenms_id = 42 → returns 42 for any server_key + New: librenms_id = {"primary": 42} → returns 42 only for server_key="primary" + + Args: + obj: NetBox object with a ``librenms_id`` custom field. + server_key: LibreNMS server key (from plugin ``servers`` config). + + Returns: + int or None + """ + cf_value = obj.cf.get("librenms_id") + if cf_value is None: + return None + if isinstance(cf_value, int): + return cf_value # backward compat: bare integer from pre-migration + if isinstance(cf_value, dict): + return cf_value.get(server_key) + return None + + +def set_librenms_device_id(obj, device_id, server_key: str = "default"): + """ + Set the LibreNMS device/port ID for a specific server on the JSON custom field. + + Migrates any legacy bare-integer value to the dict format on first write. + + Args: + obj: NetBox object with a ``librenms_id`` custom field. + device_id: LibreNMS device ID (integer). + server_key: LibreNMS server key (from plugin ``servers`` config). + """ + cf_value = obj.custom_field_data.get("librenms_id") or {} + if isinstance(cf_value, int): + cf_value = {"default": cf_value} # migrate legacy value on first write + cf_value[server_key] = device_id + obj.custom_field_data["librenms_id"] = cf_value + + +def find_by_librenms_id(model, librenms_id, server_key: str = "default"): + """ + Return the first object of *model* whose ``librenms_id`` JSON field contains + *librenms_id* under *server_key*. + + Args: + model: A Django model class (Device, VirtualMachine, Interface, …). + librenms_id: The LibreNMS device/port ID to look up. + server_key: LibreNMS server key (from plugin ``servers`` config). + + Returns: + Model instance or None + """ + return model.objects.filter(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}).first() + + def convert_speed_to_kbps(speed_bps: int) -> int: """ Convert speed from bits per second to kilobits per second. diff --git a/netbox_librenms_plugin/views/base/cables_view.py b/netbox_librenms_plugin/views/base/cables_view.py index c390cdd539..89c4a72a0e 100644 --- a/netbox_librenms_plugin/views/base/cables_view.py +++ b/netbox_librenms_plugin/views/base/cables_view.py @@ -78,10 +78,11 @@ def get_links_data(self, obj): def get_device_by_id_or_name(self, remote_device_id, hostname): """Try to find device in NetBox first by librenms_id custom field, then by name""" + server_key = self.librenms_api.server_key # First try matching by LibreNMS ID if remote_device_id: try: - device = Device.objects.get(custom_field_data__librenms_id=remote_device_id) + device = Device.objects.get(**{f"custom_field_data__librenms_id__{server_key}": remote_device_id}) return device, True, None except Device.DoesNotExist: pass @@ -116,13 +117,16 @@ def enrich_local_port(self, link, obj): if local_port := link.get("local_port"): interface = None local_port_id = link.get("local_port_id") + server_key = self.librenms_api.server_key if hasattr(obj, "virtual_chassis") and obj.virtual_chassis: chassis_member = get_virtual_chassis_member(obj, local_port) # First try to find interface by librenms_id if local_port_id: - interface = chassis_member.interfaces.filter(custom_field_data__librenms_id=local_port_id).first() + interface = chassis_member.interfaces.filter( + **{f"custom_field_data__librenms_id__{server_key}": local_port_id} + ).first() # Only if librenms_id match fails, try matching by name if not interface: @@ -130,7 +134,9 @@ def enrich_local_port(self, link, obj): else: # First try to find interface by librenms_id if local_port_id: - interface = obj.interfaces.filter(custom_field_data__librenms_id=local_port_id).first() + interface = obj.interfaces.filter( + **{f"custom_field_data__librenms_id__{server_key}": local_port_id} + ).first() # Only if librenms_id match fails, try matching by name if not interface: @@ -145,6 +151,7 @@ def enrich_remote_port(self, link, device): if remote_port := link.get("remote_port"): netbox_remote_interface = None librenms_remote_port_id = link.get("remote_port_id") + server_key = self.librenms_api.server_key # Handle virtual chassis case if hasattr(device, "virtual_chassis") and device.virtual_chassis: @@ -154,7 +161,7 @@ def enrich_remote_port(self, link, device): # First try to find interface by librenms_id if librenms_remote_port_id: netbox_remote_interface = chassis_member.interfaces.filter( - custom_field_data__librenms_id=librenms_remote_port_id + **{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id} ).first() # If not found by librenms_id, fall back to name matching on the correct chassis member @@ -165,7 +172,7 @@ def enrich_remote_port(self, link, device): # First try to find interface by librenms_id if librenms_remote_port_id: netbox_remote_interface = device.interfaces.filter( - custom_field_data__librenms_id=librenms_remote_port_id + **{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id} ).first() # If not found by librenms_id, fall back to name matching @@ -369,8 +376,9 @@ def post(self, request): # First try to find interface by librenms_id interface = None if local_port_id := link_data.get("local_port_id"): + _sk = self.librenms_api.server_key interface = selected_device.interfaces.filter( - custom_field_data__librenms_id=local_port_id + **{f"custom_field_data__librenms_id__{_sk}": local_port_id} ).first() # If not found by librenms_id, try matching by name diff --git a/netbox_librenms_plugin/views/base/ip_addresses_view.py b/netbox_librenms_plugin/views/base/ip_addresses_view.py index 22f4b49742..1ca3f6e290 100644 --- a/netbox_librenms_plugin/views/base/ip_addresses_view.py +++ b/netbox_librenms_plugin/views/base/ip_addresses_view.py @@ -11,7 +11,7 @@ from virtualization.models import VirtualMachine from netbox_librenms_plugin.tables.ipaddresses import IPAddressTable -from netbox_librenms_plugin.utils import get_interface_name_field +from netbox_librenms_plugin.utils import get_interface_name_field, get_librenms_device_id from netbox_librenms_plugin.views.mixins import CacheMixin, LibreNMSAPIMixin, LibreNMSPermissionMixin @@ -104,10 +104,11 @@ def _prefetch_netbox_data(self, obj): all_interfaces = list(obj.interfaces.all()) # Create maps for efficient lookups + server_key = self.librenms_api.server_key interfaces_by_librenms_id = { - interface.custom_field_data.get("librenms_id"): interface + get_librenms_device_id(interface, server_key): interface for interface in all_interfaces - if interface.custom_field_data.get("librenms_id") + if get_librenms_device_id(interface, server_key) } interfaces_by_name = {interface.name: interface for interface in all_interfaces} diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 696a3cc2bd..b0e262a80a 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -29,7 +29,7 @@ fetch_model_by_id, ) from netbox_librenms_plugin.tables.device_status import DeviceImportTable -from netbox_librenms_plugin.utils import get_user_pref, save_user_pref +from netbox_librenms_plugin.utils import get_user_pref, save_user_pref, set_librenms_device_id from netbox_librenms_plugin.views.mixins import LibreNMSAPIMixin, LibreNMSPermissionMixin logger = logging.getLogger(__name__) @@ -153,6 +153,7 @@ def get_validated_device_with_selections(self, device_id: int, request) -> tuple include_vc_detection=enable_vc, use_sysname=use_sysname, strip_domain=strip_domain, + server_key=self.librenms_api.server_key, ) validation["import_as_vm"] = is_vm @@ -299,6 +300,7 @@ def post(self, request): api=self.librenms_api, use_sysname=use_sysname, strip_domain=strip_domain, + server_key=self.librenms_api.server_key, ) # Mark validation with VC detection flag for proper URL generation in table @@ -664,6 +666,7 @@ def post(self, request): # noqa: PLR0912 - branching keeps responses explicit import_as_vm=is_vm, api=None, # No VC detection needed for already-imported devices include_vc_detection=False, + server_key=self.librenms_api.server_key, ) validation["import_as_vm"] = is_vm @@ -918,7 +921,7 @@ def post(self, request, device_id): strip_domain=request.POST.get("strip-domain-toggle") == "on", ) ) - existing_device.custom_field_data["librenms_id"] = int(librenms_id) + set_librenms_device_id(existing_device, int(librenms_id), self.librenms_api.server_key) existing_device.name = hostname if librenms_device_type: existing_device.device_type = librenms_device_type @@ -938,7 +941,7 @@ def post(self, request, device_id): strip_domain=request.POST.get("strip-domain-toggle") == "on", ) ) - existing_device.custom_field_data["librenms_id"] = int(librenms_id) + set_librenms_device_id(existing_device, int(librenms_id), self.librenms_api.server_key) if incoming_serial and incoming_serial != "-": conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() if conflict_device: @@ -960,7 +963,7 @@ def post(self, request, device_id): elif action == "update_serial": # Update only the serial and link to LibreNMS incoming_serial = libre_device.get("serial") or "" - existing_device.custom_field_data["librenms_id"] = int(librenms_id) + set_librenms_device_id(existing_device, int(librenms_id), self.librenms_api.server_key) if incoming_serial and incoming_serial != "-": conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() if conflict_device: diff --git a/netbox_librenms_plugin/views/object_sync/devices.py b/netbox_librenms_plugin/views/object_sync/devices.py index 97584f8319..429a3119a8 100644 --- a/netbox_librenms_plugin/views/object_sync/devices.py +++ b/netbox_librenms_plugin/views/object_sync/devices.py @@ -79,13 +79,22 @@ def get_redirect_url(self, obj): def get_table(self, data, obj, interface_name_field, vlan_groups=None): """Return the appropriate interface table, selecting VC variant if needed.""" + server_key = self.librenms_api.server_key if hasattr(obj, "virtual_chassis") and obj.virtual_chassis: table = VCInterfaceTable( - data, device=obj, interface_name_field=interface_name_field, vlan_groups=vlan_groups + data, + device=obj, + interface_name_field=interface_name_field, + vlan_groups=vlan_groups, + server_key=server_key, ) else: table = LibreNMSInterfaceTable( - data, device=obj, interface_name_field=interface_name_field, vlan_groups=vlan_groups + data, + device=obj, + interface_name_field=interface_name_field, + vlan_groups=vlan_groups, + server_key=server_key, ) table.htmx_url = f"{self.request.path}?tab=interfaces" return table diff --git a/netbox_librenms_plugin/views/object_sync/vms.py b/netbox_librenms_plugin/views/object_sync/vms.py index 51143d909d..bd6e052488 100644 --- a/netbox_librenms_plugin/views/object_sync/vms.py +++ b/netbox_librenms_plugin/views/object_sync/vms.py @@ -45,7 +45,9 @@ class VMInterfaceTableView(BaseInterfaceTableView): def get_table(self, data, obj, interface_name_field, vlan_groups=None): """Return a VM interface table for the given data.""" - return LibreNMSVMInterfaceTable(data, device=obj, vlan_groups=vlan_groups) + return LibreNMSVMInterfaceTable( + data, device=obj, vlan_groups=vlan_groups, server_key=self.librenms_api.server_key + ) def get_interfaces(self, obj): """Return all interfaces for the virtual machine.""" diff --git a/netbox_librenms_plugin/views/sync/interfaces.py b/netbox_librenms_plugin/views/sync/interfaces.py index 1100e0da1b..e4bd1ad547 100644 --- a/netbox_librenms_plugin/views/sync/interfaces.py +++ b/netbox_librenms_plugin/views/sync/interfaces.py @@ -9,7 +9,7 @@ from virtualization.models import VirtualMachine, VMInterface from netbox_librenms_plugin.models import InterfaceTypeMapping -from netbox_librenms_plugin.utils import convert_speed_to_kbps, get_interface_name_field +from netbox_librenms_plugin.utils import convert_speed_to_kbps, get_interface_name_field, set_librenms_device_id from netbox_librenms_plugin.views.mixins import ( CacheMixin, LibreNMSPermissionMixin, @@ -235,7 +235,7 @@ def update_interface_attributes( setattr(interface, netbox_key, librenms_interface.get(librenms_key)) if "librenms_id" in interface.cf: - interface.custom_field_data["librenms_id"] = librenms_interface.get("port_id") + set_librenms_device_id(interface, librenms_interface.get("port_id"), self.librenms_api.server_key) if "enabled" not in exclude_columns: admin_status = librenms_interface.get("ifAdminStatus") From b9a44b1ee74f6492f4cc6c7d44ff374d520362a8 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sat, 28 Feb 2026 22:55:32 +0100 Subject: [PATCH 09/39] fix: address librenms_id-related PR review comments - utils.py set_librenms_device_id: defensively handle unexpected types (non-int, non-dict) by resetting to empty dict with a warning log - utils.py find_by_librenms_id: add OR query to also match legacy records where librenms_id is stored as a bare integer - cables_view.py: update all 6 librenms_id lookup sites to use Q()|Q() so legacy integer records are matched alongside new JSON format - actions.py: validate librenms_id (device_id) before int() cast and return 400 if missing/non-numeric; use pre-validated int throughout the link/update/update_serial action blocks - interfaces.py: guard against None port_id before set_librenms_device_id to avoid clobbering an existing server-key mapping --- netbox_librenms_plugin/utils.py | 22 ++++++++++++++++++- .../views/base/cables_view.py | 21 +++++++++++++----- .../views/imports/actions.py | 10 ++++++--- .../views/sync/interfaces.py | 4 +++- 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/netbox_librenms_plugin/utils.py b/netbox_librenms_plugin/utils.py index aae146b187..fee1d74d4e 100644 --- a/netbox_librenms_plugin/utils.py +++ b/netbox_librenms_plugin/utils.py @@ -9,6 +9,13 @@ from utilities.paginator import get_paginate_count as netbox_get_paginate_count +import logging + +from django.db.models import Q + +logger = logging.getLogger(__name__) + + def get_librenms_device_id(obj, server_key: str = "default"): """ Get the LibreNMS device/port ID for a specific server from the JSON custom field. @@ -49,6 +56,13 @@ def set_librenms_device_id(obj, device_id, server_key: str = "default"): cf_value = obj.custom_field_data.get("librenms_id") or {} if isinstance(cf_value, int): cf_value = {"default": cf_value} # migrate legacy value on first write + elif not isinstance(cf_value, dict): + logger.warning( + "librenms_id custom field has unexpected type %s on %r; resetting to empty dict.", + type(cf_value).__name__, + obj, + ) + cf_value = {} cf_value[server_key] = device_id obj.custom_field_data["librenms_id"] = cf_value @@ -58,6 +72,9 @@ def find_by_librenms_id(model, librenms_id, server_key: str = "default"): Return the first object of *model* whose ``librenms_id`` JSON field contains *librenms_id* under *server_key*. + Also matches legacy records that stored ``librenms_id`` as a bare integer + directly in ``custom_field_data``. + Args: model: A Django model class (Device, VirtualMachine, Interface, …). librenms_id: The LibreNMS device/port ID to look up. @@ -66,7 +83,10 @@ def find_by_librenms_id(model, librenms_id, server_key: str = "default"): Returns: Model instance or None """ - return model.objects.filter(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}).first() + return model.objects.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}) + | Q(custom_field_data__librenms_id=librenms_id) + ).first() def convert_speed_to_kbps(speed_bps: int) -> int: diff --git a/netbox_librenms_plugin/views/base/cables_view.py b/netbox_librenms_plugin/views/base/cables_view.py index 89c4a72a0e..4cbcdb7a36 100644 --- a/netbox_librenms_plugin/views/base/cables_view.py +++ b/netbox_librenms_plugin/views/base/cables_view.py @@ -4,6 +4,7 @@ from django.contrib import messages from django.core.cache import cache from django.core.exceptions import MultipleObjectsReturned +from django.db.models import Q from django.http import JsonResponse from django.shortcuts import get_object_or_404, render from django.urls import reverse @@ -82,7 +83,10 @@ def get_device_by_id_or_name(self, remote_device_id, hostname): # First try matching by LibreNMS ID if remote_device_id: try: - device = Device.objects.get(**{f"custom_field_data__librenms_id__{server_key}": remote_device_id}) + device = Device.objects.get( + Q(**{f"custom_field_data__librenms_id__{server_key}": remote_device_id}) + | Q(custom_field_data__librenms_id=remote_device_id) + ) return device, True, None except Device.DoesNotExist: pass @@ -125,7 +129,8 @@ def enrich_local_port(self, link, obj): # First try to find interface by librenms_id if local_port_id: interface = chassis_member.interfaces.filter( - **{f"custom_field_data__librenms_id__{server_key}": local_port_id} + Q(**{f"custom_field_data__librenms_id__{server_key}": local_port_id}) + | Q(custom_field_data__librenms_id=local_port_id) ).first() # Only if librenms_id match fails, try matching by name @@ -135,7 +140,8 @@ def enrich_local_port(self, link, obj): # First try to find interface by librenms_id if local_port_id: interface = obj.interfaces.filter( - **{f"custom_field_data__librenms_id__{server_key}": local_port_id} + Q(**{f"custom_field_data__librenms_id__{server_key}": local_port_id}) + | Q(custom_field_data__librenms_id=local_port_id) ).first() # Only if librenms_id match fails, try matching by name @@ -161,7 +167,8 @@ def enrich_remote_port(self, link, device): # First try to find interface by librenms_id if librenms_remote_port_id: netbox_remote_interface = chassis_member.interfaces.filter( - **{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id} + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id}) + | Q(custom_field_data__librenms_id=librenms_remote_port_id) ).first() # If not found by librenms_id, fall back to name matching on the correct chassis member @@ -172,7 +179,8 @@ def enrich_remote_port(self, link, device): # First try to find interface by librenms_id if librenms_remote_port_id: netbox_remote_interface = device.interfaces.filter( - **{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id} + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id}) + | Q(custom_field_data__librenms_id=librenms_remote_port_id) ).first() # If not found by librenms_id, fall back to name matching @@ -378,7 +386,8 @@ def post(self, request): if local_port_id := link_data.get("local_port_id"): _sk = self.librenms_api.server_key interface = selected_device.interfaces.filter( - **{f"custom_field_data__librenms_id__{_sk}": local_port_id} + Q(**{f"custom_field_data__librenms_id__{_sk}": local_port_id}) + | Q(custom_field_data__librenms_id=local_port_id) ).first() # If not found by librenms_id, try matching by name diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index b0e262a80a..8da9da8b0b 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -894,6 +894,10 @@ def post(self, request, device_id): librenms_device_type = validation.get("device_type", {}).get("device_type") librenms_id = libre_device.get("device_id") + try: + librenms_id = int(librenms_id) + except (TypeError, ValueError): + return HttpResponse("Invalid or missing LibreNMS device_id in payload", status=400) # Check for LibreNMS ID collision before any linking action if action in {"link", "update", "update_serial"}: @@ -921,7 +925,7 @@ def post(self, request, device_id): strip_domain=request.POST.get("strip-domain-toggle") == "on", ) ) - set_librenms_device_id(existing_device, int(librenms_id), self.librenms_api.server_key) + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) existing_device.name = hostname if librenms_device_type: existing_device.device_type = librenms_device_type @@ -941,7 +945,6 @@ def post(self, request, device_id): strip_domain=request.POST.get("strip-domain-toggle") == "on", ) ) - set_librenms_device_id(existing_device, int(librenms_id), self.librenms_api.server_key) if incoming_serial and incoming_serial != "-": conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() if conflict_device: @@ -954,6 +957,7 @@ def post(self, request, device_id): existing_device.name = hostname if librenms_device_type: existing_device.device_type = librenms_device_type + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) existing_device.save() logger.info( f"Updated device '{existing_device.name}': serial={incoming_serial}, " @@ -963,7 +967,6 @@ def post(self, request, device_id): elif action == "update_serial": # Update only the serial and link to LibreNMS incoming_serial = libre_device.get("serial") or "" - set_librenms_device_id(existing_device, int(librenms_id), self.librenms_api.server_key) if incoming_serial and incoming_serial != "-": conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() if conflict_device: @@ -975,6 +978,7 @@ def post(self, request, device_id): existing_device.serial = incoming_serial if librenms_device_type: existing_device.device_type = librenms_device_type + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) existing_device.save() logger.info( f"Updated serial on device '{existing_device.name}' to {incoming_serial}, " diff --git a/netbox_librenms_plugin/views/sync/interfaces.py b/netbox_librenms_plugin/views/sync/interfaces.py index e4bd1ad547..62f7e9eb3a 100644 --- a/netbox_librenms_plugin/views/sync/interfaces.py +++ b/netbox_librenms_plugin/views/sync/interfaces.py @@ -235,7 +235,9 @@ def update_interface_attributes( setattr(interface, netbox_key, librenms_interface.get(librenms_key)) if "librenms_id" in interface.cf: - set_librenms_device_id(interface, librenms_interface.get("port_id"), self.librenms_api.server_key) + port_id = librenms_interface.get("port_id") + if port_id is not None: + set_librenms_device_id(interface, port_id, self.librenms_api.server_key) if "enabled" not in exclude_columns: admin_status = librenms_interface.get("ifAdminStatus") From 16dfb658781ba7f1cdc77a40d1cef91d2b41d830 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sat, 28 Feb 2026 22:57:35 +0100 Subject: [PATCH 10/39] fix: update device_filter mocks to accept Q positional args find_by_librenms_id now calls .filter(Q(...)|Q(...)) passing a positional Q object. The three TestSerialNumberMatching mocks only accepted **kwargs, causing TypeError which was silently caught by the except block in device_operations.py, making existing_device None. Update all three device_filter side_effects to accept *args and check str(arg) for 'librenms_id' so Q-based lookups are detected correctly. --- .../tests/test_import_utils.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index a537f95d1c..0f0b7a5c39 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1344,9 +1344,12 @@ def test_librenms_id_match_shows_serial_confirmed(self): self.mock_vm.objects.filter.return_value.first.return_value = None - def device_filter(**kwargs): + def device_filter(*args, **kwargs): result = MagicMock() - if any(k.startswith("custom_field_data__librenms_id") for k in kwargs): + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + if q_has_librenms: result.first.return_value = existing else: result.first.return_value = None @@ -1384,9 +1387,12 @@ def test_librenms_id_match_detects_serial_drift(self): self.mock_vm.objects.filter.return_value.first.return_value = None - def device_filter(**kwargs): + def device_filter(*args, **kwargs): result = MagicMock() - if any(k.startswith("custom_field_data__librenms_id") for k in kwargs): + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + if q_has_librenms: result.first.return_value = existing elif "serial" in kwargs: result.first.return_value = None @@ -1426,9 +1432,12 @@ def test_librenms_id_match_still_validates_site(self): self.mock_vm.objects.filter.return_value.first.return_value = None - def device_filter(**kwargs): + def device_filter(*args, **kwargs): result = MagicMock() - if any(k.startswith("custom_field_data__librenms_id") for k in kwargs): + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + if q_has_librenms: result.first.return_value = existing else: result.first.return_value = None From 167eadeed4f86521e614624497edf2188f36bc6b Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 1 Mar 2026 16:56:34 +0100 Subject: [PATCH 11/39] feat: squash-merge inventory branch onto refactor/librenms_id - Add module sync (ENTITY-MIB inventory via SNMP) from inventory branch - Resolve 5 merge conflicts: devcontainer scripts, forms.py, views/__init__.py, actions.py - Accept deletion of monolithic import_utils.py (replaced by split package) - Port _try_chassis_device_type_match() + chassis fallback to device_operations.py - Port _refresh_existing_device() rewrite to bulk_import.py (uses find_by_librenms_id for JSON compat) - Add DeviceTypeMapping/ModuleTypeMapping/NormalizationRule views to __init__.py - Server-specific cache key in poller group choices (forms.py) --- .devcontainer/README.md | 2 + .devcontainer/scripts/diagnose.sh | 1 + .devcontainer/scripts/load-aliases.sh | 1 + .devcontainer/scripts/setup.sh | 1 + .devcontainer/scripts/start-netbox.sh | 2 +- .devcontainer/scripts/welcome.sh | 3 +- .github/workflows/lint-format.yaml | 28 +- contrib/README.md | 28 + contrib/device_type_mappings.yaml | 73 ++ contrib/interface_name_rules.yaml | 200 ++++ contrib/interface_type_mappings.yaml | 70 ++ contrib/module_bay_mappings.yaml | 216 ++++ contrib/module_type_mappings.yaml | 332 +++++ contrib/normalization_rules.yaml | 61 + docs/usage_tips/custom_field.md | 8 +- docs/usage_tips/permissions.md | 2 +- netbox_librenms_plugin/__init__.py | 64 + netbox_librenms_plugin/api/serializers.py | 56 +- netbox_librenms_plugin/api/urls.py | 4 + netbox_librenms_plugin/api/views.py | 58 +- netbox_librenms_plugin/filters.py | 42 +- netbox_librenms_plugin/forms.py | 207 +++- .../import_utils/bulk_import.py | 111 +- .../import_utils/device_operations.py | 49 + netbox_librenms_plugin/librenms_api.py | 45 + .../migrations/0009_add_devicetypemapping.py | 45 + .../migrations/0010_add_moduletypemapping.py | 49 + .../migrations/0011_modulebaymapping.py | 38 + .../0012_add_is_regex_to_modulebaymapping.py | 18 + .../migrations/0013_normalizationrule.py | 93 ++ netbox_librenms_plugin/models.py | 203 ++++ netbox_librenms_plugin/navigation.py | 68 ++ .../js/librenms_sync.js | 4 + netbox_librenms_plugin/tables/mappings.py | 137 ++- netbox_librenms_plugin/tables/modules.py | 180 +++ .../_module_sync_content.html | 31 + .../devicetypemapping.html | 28 + .../devicetypemapping_list.html | 12 + .../htmx/device_validation_details.html | 2 +- .../inc/_module_sync.html | 27 + .../librenms_sync_base.html | 19 +- .../modulebaymapping.html | 30 + .../modulebaymapping_list.html | 12 + .../moduletypemapping.html | 28 + .../moduletypemapping_list.html | 12 + .../normalizationrule.html | 34 + .../normalizationrule_list.html | 16 + netbox_librenms_plugin/tests/test_init.py | 171 +++ netbox_librenms_plugin/tests/test_utils.py | 31 +- netbox_librenms_plugin/urls.py | 220 +++- netbox_librenms_plugin/utils.py | 134 ++- netbox_librenms_plugin/views/__init__.py | 34 + .../views/base/cables_view.py | 11 +- .../views/base/librenms_sync_view.py | 11 + .../views/base/modules_view.py | 1064 +++++++++++++++++ netbox_librenms_plugin/views/mapping_views.py | 276 ++++- .../views/object_sync/__init__.py | 1 + .../views/object_sync/devices.py | 20 + netbox_librenms_plugin/views/sync/cables.py | 29 +- .../views/sync/device_fields.py | 40 +- netbox_librenms_plugin/views/sync/devices.py | 2 +- .../views/sync/interfaces.py | 6 - tests/e2e/__init__.py | 0 tests/e2e/conftest.py | 6 + tests/e2e/test_module_install.py | 330 +++++ 65 files changed, 5007 insertions(+), 129 deletions(-) create mode 100644 contrib/README.md create mode 100644 contrib/device_type_mappings.yaml create mode 100644 contrib/interface_name_rules.yaml create mode 100644 contrib/interface_type_mappings.yaml create mode 100644 contrib/module_bay_mappings.yaml create mode 100644 contrib/module_type_mappings.yaml create mode 100644 contrib/normalization_rules.yaml create mode 100644 netbox_librenms_plugin/migrations/0009_add_devicetypemapping.py create mode 100644 netbox_librenms_plugin/migrations/0010_add_moduletypemapping.py create mode 100644 netbox_librenms_plugin/migrations/0011_modulebaymapping.py create mode 100644 netbox_librenms_plugin/migrations/0012_add_is_regex_to_modulebaymapping.py create mode 100644 netbox_librenms_plugin/migrations/0013_normalizationrule.py create mode 100644 netbox_librenms_plugin/tables/modules.py create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping_list.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/inc/_module_sync.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping_list.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping_list.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule_list.html create mode 100644 netbox_librenms_plugin/tests/test_init.py create mode 100644 netbox_librenms_plugin/views/base/modules_view.py create mode 100644 tests/e2e/__init__.py create mode 100644 tests/e2e/conftest.py create mode 100644 tests/e2e/test_module_install.py diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 3560a93af3..7024351758 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -98,6 +98,8 @@ Below are the dev container defaults. The field name to change these defaults is - Plugin loader: enabled; reads `.devcontainer/config/plugin-config.py` if present - If `plugin-config.py` is missing: plugin is enabled with empty config (features won’t work until configured) + + ## 🔧 Configuration ### NetBox Version and Environment (use .devcontainer/.env) diff --git a/.devcontainer/scripts/diagnose.sh b/.devcontainer/scripts/diagnose.sh index 133e6ca97f..be7596d699 100755 --- a/.devcontainer/scripts/diagnose.sh +++ b/.devcontainer/scripts/diagnose.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script echo "🔍 DevContainer Startup Diagnostics" echo "==================================" diff --git a/.devcontainer/scripts/load-aliases.sh b/.devcontainer/scripts/load-aliases.sh index 65149d6198..feac6ee98f 100755 --- a/.devcontainer/scripts/load-aliases.sh +++ b/.devcontainer/scripts/load-aliases.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script # Quick alias loader for current session # Usage: source .devcontainer/scripts/load-aliases.sh diff --git a/.devcontainer/scripts/setup.sh b/.devcontainer/scripts/setup.sh index 588fe652ab..7f4278fd46 100755 --- a/.devcontainer/scripts/setup.sh +++ b/.devcontainer/scripts/setup.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script set -e echo "🚀 Setting up NetBox LibreNMS Plugin development environment..." diff --git a/.devcontainer/scripts/start-netbox.sh b/.devcontainer/scripts/start-netbox.sh index 789dcb845a..d5e4796600 100755 --- a/.devcontainer/scripts/start-netbox.sh +++ b/.devcontainer/scripts/start-netbox.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script # Check if we should run in background or foreground BACKGROUND=false @@ -18,7 +19,6 @@ if [ "$CODESPACES" = "true" ] && [ -n "$CODESPACE_NAME" ]; then echo "🔗 GitHub Codespaces detected" else ACCESS_URL="http://localhost:8000" - echo "🐛 Debug: ACCESS_URL is set to: $ACCESS_URL" fi # Load shared process management helpers diff --git a/.devcontainer/scripts/welcome.sh b/.devcontainer/scripts/welcome.sh index 9328d663aa..e273313766 100755 --- a/.devcontainer/scripts/welcome.sh +++ b/.devcontainer/scripts/welcome.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script # Ensure aliases are available in the postAttach terminal session source "$(dirname "$0")/load-aliases.sh" 2>/dev/null @@ -44,7 +45,7 @@ if [ -n "$CODESPACES" ]; then echo " 💡 Click the link in the Ports panel or look for the 'Open in Browser' button" else echo "🖥️ Local Development Environment:" - echo " NetBox will be available at: http://localhost:8000 (paste into you browser)" + echo " NetBox will be available at: http://localhost:8000 (paste into your browser)" fi echo "" diff --git a/.github/workflows/lint-format.yaml b/.github/workflows/lint-format.yaml index 3e12242f63..055f809cc5 100644 --- a/.github/workflows/lint-format.yaml +++ b/.github/workflows/lint-format.yaml @@ -2,13 +2,7 @@ name: Lint and Format on: push: - branches: - - master - - develop pull_request: - branches: - - master - - develop jobs: format-and-lint: @@ -20,8 +14,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: '3.9' - cache: 'pip' + python-version: '3.12' - name: Install dependencies run: | @@ -29,22 +22,7 @@ jobs: pip install ruff - name: Run Ruff linting - run: | - echo "::group::Ruff Linting" - ruff check . --output-format=github - echo "::endgroup::" + run: ruff check . - name: Run Ruff formatting check - run: | - echo "::group::Ruff Formatting" - ruff format --check . - echo "::endgroup::" - - - name: Report formatting issues - if: failure() - run: | - echo "::error::Formatting or linting issues detected!" - echo "To fix locally, run:" - echo " ruff check --fix ." - echo " ruff format ." - echo "Then commit and push the changes." + run: ruff format --check . diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000000..8714ac5342 --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,28 @@ +# Contrib: Example Mapping Files + +This directory contains example YAML mapping files for bulk import into the +NetBox LibreNMS Plugin. Each file can be imported via the plugin's bulk import +feature in the NetBox UI. + +## How to Import + +1. Navigate to the mapping page (e.g., **LibreNMS → Device Type Mappings**) +2. Click the **Import** button (upload icon) in the top right +3. Select **YAML** format +4. Paste the contents of the relevant YAML file +5. Click **Submit** + +## Available Mappings + +| File | Description | +|------|-------------| +| `interface_type_mappings.yaml` | Maps LibreNMS interface types + speeds to NetBox interface types | +| `device_type_mappings.yaml` | Maps LibreNMS hardware strings to NetBox device types | +| `module_type_mappings.yaml` | Maps LibreNMS inventory model names to NetBox module types (incl. transceivers) | +| `module_bay_mappings.yaml` | Maps LibreNMS inventory container names to NetBox module bay names | + +## Customisation + +These files are **examples** — adjust values to match the device types, module +types, and interface types defined in your NetBox instance. The `netbox_*` +fields must reference objects that already exist in your NetBox. diff --git a/contrib/device_type_mappings.yaml b/contrib/device_type_mappings.yaml new file mode 100644 index 0000000000..2dec241524 --- /dev/null +++ b/contrib/device_type_mappings.yaml @@ -0,0 +1,73 @@ +# Device Type Mappings +# +# Maps LibreNMS hardware strings to NetBox device types. +# Import via: LibreNMS Plugin > Device Type Mappings > Import +# +# Fields: +# librenms_hardware — Hardware string exactly as shown in LibreNMS +# netbox_device_type — NetBox DeviceType (matched by model name or ID) +# description — Optional note +# +# The librenms_hardware value is matched case-insensitively. +# These mappings are checked BEFORE the built-in part_number/model fallback. + +# Juniper — LibreNMS reports verbose marketing names +- librenms_hardware: "Juniper MX480 Internet Backbone Router" + netbox_device_type: "MX480" + description: "Juniper MX480 chassis" + +- librenms_hardware: "Juniper MX960 Internet Backbone Router" + netbox_device_type: "MX960" + description: "Juniper MX960 chassis" + +- librenms_hardware: "Juniper MX304 Edge Router" + netbox_device_type: "MX304" + description: "Juniper MX304 edge router" + +- librenms_hardware: "JNP10008 [PTX10008]" + netbox_device_type: "PTX10008" + description: "Juniper PTX10008 core router" + +- librenms_hardware: "JNP7100-32C [ACX7100-32C]" + netbox_device_type: "ACX7100-32C" + description: "Juniper ACX7100-32C" + +- librenms_hardware: "JNP7024 [ACX7024]" + netbox_device_type: "ACX7024" + description: "Juniper ACX7024" + +- librenms_hardware: "Juniper JNP10008 Internet Backbone Router" + netbox_device_type: "PTX10008" + description: "Juniper PTX10008 (alternate hardware string)" + +- librenms_hardware: "Juniper VRR Internet Backbone Router" + netbox_device_type: "VRR" + description: "Juniper Virtual Route Reflector" + +# Nokia — model string matches directly in most cases +- librenms_hardware: "7750 SR-7s" + netbox_device_type: "7750 SR-7s" + description: "Nokia 7750 SR-7s service router" + +# Cisco — often matches by part_number but not always +- librenms_hardware: "WS-C4900M" + netbox_device_type: "WS-C4900M" + description: "Cisco Catalyst 4900M" + +# Cisco IOS XR +- librenms_hardware: "8201-SYS" + netbox_device_type: "8201" + description: "Cisco 8201 (hardware string differs from model)" + +# UfiSpace — LibreNMS reports SONiC/ONIE platform names +- librenms_hardware: "x86-64-ufispace-s9610-36d-r0" + netbox_device_type: "S9610-36D" + description: "UfiSpace S9610-36D" + +- librenms_hardware: "x86-64-ufispace-s9610-46dx-r0" + netbox_device_type: "S9610-46DX" + description: "UfiSpace S9610-46DX" + +- librenms_hardware: "x86-64-ufispace-s9700-53dx-r9" + netbox_device_type: "S9700-53DX" + description: "UfiSpace S9700-53DX" diff --git a/contrib/interface_name_rules.yaml b/contrib/interface_name_rules.yaml new file mode 100644 index 0000000000..52da69dff5 --- /dev/null +++ b/contrib/interface_name_rules.yaml @@ -0,0 +1,200 @@ +# Interface Name Rules +# +# Post-install interface rename rules for module types where NetBox's +# position-based naming can't produce the correct interface name. +# +# Covers two scenarios: +# 1. Converter offset — e.g., GLC-T inside CVR-X2-SFP needs port numbering +# that accounts for the converter's position in the parent module bay. +# 2. Breakout channels — e.g., QSFP+ 4x10G produces multiple sub-interfaces +# from a single physical port. +# +# Template variables: +# {slot} — Top-level slot/module bay position +# {bay_position} — Position of the bay this module is installed into (raw) +# {bay_position_num} — Numeric suffix of bay position (e.g., "swp1" → "1") +# {parent_bay_position} — Position of the parent module's bay +# {sfp_slot} — Numeric sub-bay index within the parent module +# {base} — Original interface name from the NetBox module template +# {channel} — Breakout channel number (iterated) +# +# Arithmetic expressions are supported inside braces: +# {8 + ({parent_bay_position} - 1) * 2 + {sfp_slot}} +# +# Bulk import via: LibreNMS Plugin > Settings > Interface Name Rules > Import + +# --- Converter Offset Examples --- + +# SFP-1G-T (1G copper SFP, covers GLC-T/GLC-TE) in CVR-X2-SFP converter +# X2 bays are numbered 1-N; each converter holds 2 SFP slots +# Resulting interface: GigabitEthernet/ +- module_type: SFP-1G-T + parent_module_type: CVR-X2-SFP + name_template: "GigabitEthernet{slot}/{8 + ({parent_bay_position} - 1) * 2 + {sfp_slot}}" + channel_count: 0 + channel_start: 0 + description: "SFP-1G-T in CVR-X2-SFP: offset port numbering for X2-to-SFP conversion" + +# --- Breakout Channel Examples --- + +# QSFP-4X10G-LR breakout — Juniper-style (channels start at 0) +- module_type: QSFP-4X10G-LR + name_template: "{base}:{channel}" + channel_count: 4 + channel_start: 0 + description: "QSFP+ 4x10G-LR breakout with Juniper-style channel numbering (0-3)" + +# QSFP-4X10G-SR breakout — Juniper-style (channels start at 0) +- module_type: QSFP-4X10G-SR + name_template: "{base}:{channel}" + channel_count: 4 + channel_start: 0 + description: "QSFP+ 4x10G-SR breakout with Juniper-style channel numbering (0-3)" + +# --- Commented Examples --- + +# QSFP+ 4x10G breakout — Cisco-style (channels start at 1) +# - module_type: QSFP-4X10G-LR +# name_template: "{base}:{channel}" +# channel_count: 4 +# channel_start: 1 +# description: "QSFP+ 4x10G breakout with Cisco-style channel numbering (1-4)" + +# --- UfiSpace/Arcos Breakout Rules --- +# UfiSpace switches use swpNsC naming for breakout interfaces. +# bay_position_num extracts the numeric suffix from the bay name (e.g., "swp1" → "1"). +# Channels start at 1, with 2 channels per 100G QSFP28 (2x100G breakout). + +# S9610-36D breakout rules +- module_type: QSFP-100G-LR4 + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SR4 + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SWDM4 + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-ZR + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" + +# S9610-46DX breakout rules +- module_type: QSFP-100G-LR4 + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SR4 + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SWDM4 + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-ZR + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" + +# S9700-53DX breakout rules +- module_type: QSFP-100G-LR4 + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SR4 + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SWDM4 + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-ZR + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" + +# --- Juniper ACX7024 Platform-Specific Rules --- +# These rules are scoped to the ACX7024 device type and use bay_position +# to generate Juniper-style interface names with FPC/PIC/port notation. + +# 100GE QSFP28 transceivers -> et-0/0/{port} +- module_type: QSFP-100G-LR4 + device_type: ACX7024 + name_template: "et-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 100GE QSFP28 naming" + +- module_type: QSFP-100G-SR4 + device_type: ACX7024 + name_template: "et-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 100GE QSFP28 naming" + +- module_type: QSFP-100G-SWDM4 + device_type: ACX7024 + name_template: "et-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 100GE QSFP28 naming" + +# 10GE SFP+ transceivers -> xe-0/0/{port} +- module_type: SFP-10G-SR + device_type: ACX7024 + name_template: "xe-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 10GE SFP+ naming" + +- module_type: SFP-10G-LR + device_type: ACX7024 + name_template: "xe-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 10GE SFP+ naming" + +# 1GE SFP transceivers -> ge-0/0/{port} +- module_type: SFP-1G-T + device_type: ACX7024 + name_template: "ge-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 1GE SFP naming" + +- module_type: SFP-1G-LX + device_type: ACX7024 + name_template: "ge-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 1GE SFP naming" diff --git a/contrib/interface_type_mappings.yaml b/contrib/interface_type_mappings.yaml new file mode 100644 index 0000000000..19db2a1fcf --- /dev/null +++ b/contrib/interface_type_mappings.yaml @@ -0,0 +1,70 @@ +# Interface Type Mappings +# +# Maps LibreNMS interface types (and optional speeds) to NetBox interface types. +# Import via: LibreNMS Plugin > Interface Mappings > Import +# +# Fields: +# librenms_type — IANA ifType string from LibreNMS (e.g. ethernetCsmacd) +# librenms_speed — Speed in Kbps (optional, null matches any speed) +# netbox_type — NetBox InterfaceTypeChoices slug +# description — Optional note +# +# Common NetBox interface type slugs: +# 1000base-t, 10gbase-t, 10gbase-x-sfpp, 25gbase-x-sfp28, +# 40gbase-x-qsfpp, 100gbase-x-qsfp28, 400gbase-x-qsfpdd, +# ieee802.11ax, lag, virtual, other + +- librenms_type: ethernetCsmacd + librenms_speed: 1000000 + netbox_type: 1000base-t + description: "1G Ethernet copper" + +- librenms_type: ethernetCsmacd + librenms_speed: 10000000 + netbox_type: 10gbase-x-sfpp + description: "10G Ethernet SFP+" + +- librenms_type: ethernetCsmacd + librenms_speed: 25000000 + netbox_type: 25gbase-x-sfp28 + description: "25G Ethernet SFP28" + +- librenms_type: ethernetCsmacd + librenms_speed: 40000000 + netbox_type: 40gbase-x-qsfpp + description: "40G Ethernet QSFP+" + +- librenms_type: ethernetCsmacd + librenms_speed: 100000000 + netbox_type: 100gbase-x-qsfp28 + description: "100G Ethernet QSFP28" + +- librenms_type: ethernetCsmacd + librenms_speed: 400000000 + netbox_type: 400gbase-x-qsfpdd + description: "400G Ethernet QSFP-DD" + +- librenms_type: ieee8023adLag + librenms_speed: + netbox_type: lag + description: "LACP/LAG aggregation" + +- librenms_type: propVirtual + librenms_speed: + netbox_type: virtual + description: "Virtual/loopback interface" + +- librenms_type: softwareLoopback + librenms_speed: + netbox_type: virtual + description: "Software loopback" + +- librenms_type: tunnel + librenms_speed: + netbox_type: virtual + description: "Tunnel interface" + +- librenms_type: l2vlan + librenms_speed: + netbox_type: virtual + description: "VLAN interface" diff --git a/contrib/module_bay_mappings.yaml b/contrib/module_bay_mappings.yaml new file mode 100644 index 0000000000..64c063176a --- /dev/null +++ b/contrib/module_bay_mappings.yaml @@ -0,0 +1,216 @@ +# Module Bay Mappings - Map LibreNMS inventory container names to NetBox module bay names +# +# These mappings replace heuristic matching between LibreNMS inventory and NetBox module bays. +# Import via: LibreNMS Plugin → Module Bay Mappings → Import +# +# Fields: +# librenms_name: LibreNMS entPhysicalName or container name (exact match or regex) +# librenms_class: Optional entPhysicalClass filter (powerSupply, fan, module, etc.) +# Leave empty for class-independent mappings +# netbox_bay_name: Target NetBox module bay name (supports \1, \2 backreferences with regex) +# is_regex: Set to true to treat librenms_name as a Python regex pattern +# description: Optional description +# +# Regex patterns use Python re.fullmatch() — the pattern must match the entire string. +# Backreferences (\1, \2) in netbox_bay_name reference capture groups in the pattern. + +# ─── Regex Patterns ────────────────────────────────────────────────────────── +# These patterns replace many individual exact-match entries. + +# Arcos/UfiSpace: sfpN → Transceiver N (covers sfp0 through sfp53+) +- librenms_name: "^sfp(\\d+)$" + netbox_bay_name: "Transceiver \\1" + is_regex: true + description: "Arcos sfpN → Transceiver N" + +# Cisco X2: Port Container slot/port → X2 Port port +- librenms_name: "^Port Container (\\d+)/(\\d+)$" + netbox_bay_name: "X2 Port \\2" + is_regex: true + description: "Cisco X2 Port Container → X2 Port N" + +# Cisco modules: Linecard/Supervisor(slot N) → Slot N +- librenms_name: "^Linecard\\(slot (\\d+)\\)$" + librenms_class: "module" + netbox_bay_name: "Slot \\1" + is_regex: true + description: "Cisco Linecard slot → Slot N" +- librenms_name: "^Supervisor\\(slot (\\d+)\\)$" + librenms_class: "module" + netbox_bay_name: "Slot \\1" + is_regex: true + description: "Cisco Supervisor slot → Slot N" + +# Generic power supplies and fans +- librenms_name: "^Power Supply (\\d+)$" + librenms_class: "powerSupply" + netbox_bay_name: "PS\\1" + is_regex: true + description: "Power Supply N → PSN" +- librenms_name: "^FanTray (\\d+)$" + librenms_class: "fan" + netbox_bay_name: "Fan Tray \\1" + is_regex: true + description: "FanTray N → Fan Tray N" + +# Nokia 7750 SR chassis fans and power modules +- librenms_name: "^Chassis 1 Fan (\\d+)$" + librenms_class: "fan" + netbox_bay_name: "Fan \\1" + is_regex: true + description: "Nokia chassis fan → Fan N" +- librenms_name: "^Chassis 1 PowShelf 1 PM (\\d+)$" + librenms_class: "powerSupply" + netbox_bay_name: "PM \\1" + is_regex: true + description: "Nokia power module → PM N" + +# Nokia MDA and XIOM sub-module bays +# Bay names resolve from {module}/N templates: IOM Slot 1 pos=1 → bay {module}/1 = 1/1 +- librenms_name: "^MDA (\\d+)/(\\d+)$" + librenms_class: "mdaModule" + netbox_bay_name: "\\1/\\2" + is_regex: true + description: "Nokia MDA N/M → N/M (matches {module}/M on IOM)" +- librenms_name: "^XIOM (\\d+)/x(\\d+)$" + librenms_class: "xioModule" + netbox_bay_name: "\\1/x\\2" + is_regex: true + description: "Nokia XIOM N/xM → N/xM (matches {module}/xM on IOM)" +- librenms_name: "^MDA (\\d+)/x(\\d+)/(\\d+)$" + librenms_class: "mdaModule" + netbox_bay_name: "x\\2/\\3" + is_regex: true + description: "Nokia MDA in XIOM N/xP/Q → xP/Q (matches {module}/Q on XIOM)" + +# Nokia transceiver connector bays +# LibreNMS ifName "1/1/c1" (slot/mda/connector) → NetBox bay "1/c1" +# ({module} on MDA resolves to position, stripping the slot prefix) +- librenms_name: "(\\d+)/(\\d+)/(c\\d+)" + librenms_class: "port" + netbox_bay_name: "\\2/\\3" + is_regex: true + description: "Nokia transceiver slot/mda/cN → mda-pos/cN" +# LibreNMS ifName "2/x1/1/c2" (slot/xiom/mda/connector) → NetBox bay "1/c2" +- librenms_name: "(\\d+)/x(\\d+)/(\\d+)/(c\\d+)" + librenms_class: "port" + netbox_bay_name: "\\3/\\4" + is_regex: true + description: "Nokia XIOM transceiver slot/xiom/mda/cN → mda-pos/cN" + +# Juniper MX transceiver bays +# LibreNMS entPhysicalDescr format: "SFP+-10G-SR @ {fpc}/{pic}/{port}" +# NetBox MPC-3D-16XGE-SFPP bay format: "Transceiver {pic}/{port}" +- librenms_name: "[^@]+ @ \\d+/(\\d+)/(\\d+)" + librenms_class: "port" + netbox_bay_name: "Transceiver \\1/\\2" + is_regex: true + description: "Juniper MX SFP+ @ fpc/pic/port → Transceiver pic/port" + +# ─── Exact Match Entries ───────────────────────────────────────────────────── +# These are for special cases where names don't follow a regex pattern. + +# Nokia CPM slots +- librenms_name: "Slot A" + librenms_class: "cpmModule" + netbox_bay_name: "Slot A" + description: "Nokia CPM slot A" +- librenms_name: "Slot B" + librenms_class: "cpmModule" + netbox_bay_name: "Slot B" + description: "Nokia CPM slot B" +- librenms_name: "SR-7s 2 CPM mini" + librenms_class: "cpmCarrier" + netbox_bay_name: "CMA" + description: "Nokia CMA2-7s CPM carrier bracket" + +# Juniper fixed-form devices +- librenms_name: "PSM 0" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 0" + description: "Juniper PSU slot 0" +- librenms_name: "PSM 1" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 1" + description: "Juniper PSU slot 1" + +# Juniper chassis devices (PTX10008 etc.): PSM → PEM +# Regex runs after exact matches, so PSM 0/1 → PSU 0/1 above takes priority for ACX +- librenms_name: "^PSM (\\d+)$" + librenms_class: "powerSupply" + netbox_bay_name: "PEM \\1" + is_regex: true + description: "Juniper chassis PSM N → PEM N" + +# Juniper FPC container: "FPC: @ N/*/*" → FPC N +- librenms_name: "^FPC: .+ @ (\\d+)/\\*/\\*$" + librenms_class: "container" + netbox_bay_name: "FPC \\1" + is_regex: true + description: "Juniper FPC container description → FPC N" + +# Juniper transceivers: " @ slot/pic/port" description → Transceiver slot/pic/port +- librenms_name: "^.+ @ (\\d+/\\d+/\\d+)$" + librenms_class: "port" + netbox_bay_name: "Transceiver \\1" + is_regex: true + description: "Juniper transceiver description → Transceiver slot/pic/port" + +# Juniper fan trays: "Fan Tray N" → "Fan N" (ACX7100, etc.) +# Runs after exact match, so "Fan Tray 0" → "Fan Tray" (ACX7024) still works +- librenms_name: "^Fan Tray (\\d+)$" + librenms_class: "fan" + netbox_bay_name: "Fan \\1" + is_regex: true + description: "Juniper Fan Tray N → Fan N (ACX7100 etc.)" + +# Juniper MX304: PEM → PSU (MX304 bays are named PSU, not PEM) +- librenms_name: "PEM 0" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 0" + description: "Juniper MX304 PEM 0 → PSU 0" +- librenms_name: "PEM 1" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 1" + description: "Juniper MX304 PEM 1 → PSU 1" + +- librenms_name: "Fan Tray 0" + librenms_class: "fan" + netbox_bay_name: "Fan Tray" + description: "Juniper single fan tray (ACX7024)" + +# Juniper PTX10008: SIB → CB (Switch Interface Board → Component Board slot) +- librenms_name: "SIB 0" + librenms_class: "container" + netbox_bay_name: "CB 0" + description: "Juniper PTX10008 SIB 0 → CB 0" +- librenms_name: "SIB 1" + librenms_class: "container" + netbox_bay_name: "CB 1" + description: "Juniper PTX10008 SIB 1 → CB 1" +- librenms_name: "SIB 2" + librenms_class: "container" + netbox_bay_name: "CB 2" + description: "Juniper PTX10008 SIB 2 → CB 2" +- librenms_name: "SIB 3" + librenms_class: "container" + netbox_bay_name: "CB 3" + description: "Juniper PTX10008 SIB 3 → CB 3" +- librenms_name: "SIB 4" + librenms_class: "container" + netbox_bay_name: "CB 4" + description: "Juniper PTX10008 SIB 4 → CB 4" +- librenms_name: "SIB 5" + librenms_class: "container" + netbox_bay_name: "CB 5" + description: "Juniper PTX10008 SIB 5 → CB 5" + +# Arcos power supplies +- librenms_name: "psu0" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 0" + description: "Arcos PSU slot 0" +- librenms_name: "psu1" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 1" + description: "Arcos PSU slot 1" diff --git a/contrib/module_type_mappings.yaml b/contrib/module_type_mappings.yaml new file mode 100644 index 0000000000..e70d726f1b --- /dev/null +++ b/contrib/module_type_mappings.yaml @@ -0,0 +1,332 @@ +# Module Type Mappings +# +# Maps LibreNMS inventory model names (entPhysicalModelName) to NetBox module types. +# Import via: LibreNMS Plugin > Module Type Mappings > Import +# +# Fields: +# librenms_model — Model name from LibreNMS SNMP inventory +# netbox_module_type — NetBox ModuleType (matched by model name or ID) +# description — Optional note +# +# These mappings are checked FIRST. If no mapping exists, the plugin falls back +# to exact model name and part_number matching against NetBox module types. + +# ─── Cisco Catalyst 4900M ──────────────────────────────────────────────────── + +- librenms_model: "WS-X4908-10GE" + netbox_module_type: "WS-X4908-10GE" + description: "Cisco 8-port 10G X2 line card" + +- librenms_model: "WS-X4992" + netbox_module_type: "WS-X4992" + description: "Cisco 48-port 10/100/1000 line card" + +- librenms_model: "PWR-C49M-1000AC" + netbox_module_type: "PWR-C49M-1000AC" + description: "Cisco 1000W AC power supply" + +- librenms_model: "CVR-X2-SFP" + netbox_module_type: "CVR-X2-SFP" + description: "Cisco X2-to-SFP converter" + +# ─── Juniper Backplane ─────────────────────────────────────────────────────── + +- librenms_model: "710-017414" + netbox_module_type: "MX480-CHASSIS-BP" + description: "Juniper MX480 backplane (matched by part number)" + +- librenms_model: "CHAS-BP-MX480-S" + netbox_module_type: "MX480-CHASSIS-BP" + description: "Juniper MX480 backplane (matched by name)" + +# ─── Juniper FPC / Line Card Mappings ──────────────────────────────────────── +# Juniper FPCs use 750-xxxxxx part numbers as entPhysicalModelName. + +- librenms_model: "750-018124" + netbox_module_type: "DPCE-R-4XGE-XFP" + description: "Juniper DPCE 4-port 10G XFP DPC" + +- librenms_model: "750-022765" + netbox_module_type: "DPCE-R-20GE-2XGE" + description: "Juniper DPCE 20x1G + 2x10G combo DPC" + +- librenms_model: "750-028467" + netbox_module_type: "MPC-3D-16XGE-SFPP" + description: "Juniper MPC 16-port 10G SFP+" + +- librenms_model: "750-056519" + netbox_module_type: "MPC7E-MRATE" + description: "Juniper MPC7E 12-port QSFP+/QSFP28 multirate" + +- librenms_model: "750-062581" + netbox_module_type: "MPC-3D-16XGE-SFPP" + description: "Juniper MPC 16-port 10G SFP+ (variant PN)" + +# ─── Juniper Power Supply Mappings ─────────────────────────────────────────── + +- librenms_model: "740-029970" + netbox_module_type: "PWR-MX480-2520-AC" + description: "Juniper MX480 2520W AC PSU" + +- librenms_model: "740-063046" + netbox_module_type: "PWR-MX480-2520-AC" + description: "Juniper MX480 2520W AC PSU (variant PN)" + +- librenms_model: "740-027760" + netbox_module_type: "PWR-MX960-4100-AC" + description: "Juniper MX960 4100W AC PSU" + +- librenms_model: "740-110419" + netbox_module_type: "JNP-PWR2200-AC" + description: "Juniper MX304 2200W AC PSU" + +# Removed: JPSU-1600W-1UACAFO — exact model match, no mapping needed + +# ─── Juniper Fan Tray Mappings ─────────────────────────────────────────────── + +- librenms_model: "740-031521" + netbox_module_type: "FFANTRAY-MX960-HC" + description: "Juniper MX960 high-capacity fan tray" + +- librenms_model: "760-126744" + netbox_module_type: "JNP-FAN-2RU" + description: "Juniper MX304 2RU fan tray" + +# Removed: JNP7100-FAN1RU-AO — exact model match, no mapping needed + +# ─── Nokia 7750 SR-7s Module Mappings ──────────────────────────────────────── +# Nokia 3HE part numbers are handled by NormalizationRule: +# 1. Strip extra text (e.g. "3HE10550AARA01 NOK IPU3BFUEAA" → "3HE10550AARA01") +# 2. Strip revision suffix (e.g. "3HE10550AARA01" → "3HE10550AA") +# The normalized value matches the part_number field on NetBox ModuleTypes. +# No explicit Nokia mappings are needed. + +# ─── Transceiver Mappings: Juniper Part Numbers ───────────────────────────── +# Juniper-qualified optics use 740-xxxxxx part numbers regardless of OEM vendor. + +- librenms_model: "740-013111" + netbox_module_type: "SFP-1G-T" + description: "Juniper SFP 1000BASE-T copper" + +- librenms_model: "740-021308" + netbox_module_type: "SFP-10G-SR" + description: "Juniper SFP+ 10G-SR" + +- librenms_model: "740-031850" + netbox_module_type: "SFP-1G-LX" + description: "Juniper SFP 1000BASE-LX 10km" + +- librenms_model: "740-031980" + netbox_module_type: "SFP-10G-SR" + description: "Juniper SFP+ 10G-SR" + +- librenms_model: "740-031981" + netbox_module_type: "SFP-10G-LR" + description: "Juniper SFP+ 10G-LR" + +- librenms_model: "740-047682" + netbox_module_type: "CFP-100G-LR4" + description: "Juniper CFP 100G-LR4" + +- librenms_model: "740-054050" + netbox_module_type: "QSFP-4X10G-LR" + description: "Juniper QSFP+ 4x10G-LR" + +- librenms_model: "740-054053" + netbox_module_type: "QSFP-4X10G-SR" + description: "Juniper QSFP+ 4x10G-SR" + +- librenms_model: "740-058732" + netbox_module_type: "QSFP-100G-LR4" + description: "Juniper QSFP28 100G-LR4" + +- librenms_model: "740-061405" + netbox_module_type: "QSFP-100G-SR4" + description: "Juniper QSFP28 100G-SR4" + +- librenms_model: "740-061409" + netbox_module_type: "QSFP-100G-LR4" + description: "Juniper QSFP28 100G-LR4" + +- librenms_model: "740-079871" + netbox_module_type: "QSFP28-DD-2X100G-LR4" + description: "Juniper QSFP-DD 2x100G-LR4" + +- librenms_model: "740-082823" + netbox_module_type: "QSFP-DD-400G-LR8" + description: "Juniper QSFP-DD 400G-LR8" + +- librenms_model: "740-085349" + netbox_module_type: "QSFP-DD-400G-FR4" + description: "Juniper QSFP-DD 400G-FR4" + +- librenms_model: "740-085351" + netbox_module_type: "QSFP-DD-400G-DR4" + description: "Juniper QSFP-DD 400G-DR4" + +- librenms_model: "740-096176" + netbox_module_type: "QSFP-DD-400G-LR4" + description: "Juniper QSFP-DD 400G-LR4 (10km variant)" + +- librenms_model: "740-131169" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Juniper QSFP-DD 400G-ZR-M" + +- librenms_model: "740-151745" + netbox_module_type: "QSFP-DD-400G-ZR-M-HP" + description: "Juniper QSFP-DD 400G-ZR-M high-power" + +- librenms_model: "740-172665" + netbox_module_type: "QSFP-100G-ZR" + description: "Juniper QSFP28 100G-ZR" + +# ─── Transceiver Mappings: Finisar / II-VI / Coherent ──────────────────────── +# These are BASE part numbers (after normalization strips customer suffixes). +# See contrib/normalization_rules.yaml for the Finisar suffix-stripping rule. + +- librenms_model: "FTLC1154RDPL" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4" + +- librenms_model: "FTLC1151RDPL" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4 (variant)" + +- librenms_model: "FTLX1474D3BCL" + netbox_module_type: "SFP-10G-LR" + description: "Finisar SFP+ 10G-LR" + +- librenms_model: "FTCD3323R1PCL" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Finisar/II-VI QSFP-DD 400G-ZR-M coherent" + +- librenms_model: "FTLC9152RGPL" + netbox_module_type: "QSFP-100G-SWDM4" + description: "Finisar QSFP28 100G-SWDM4" + +# ─── Transceiver Mappings: Cisco / Cisco-branded OEM ───────────────────────── + +- librenms_model: "X2-10GB-LR" + netbox_module_type: "X2-10GB-LR" + description: "Cisco X2 10G-LR" + +- librenms_model: "X2-10GB-SR" + netbox_module_type: "X2-10GB-SR" + description: "Cisco X2 10G-SR" + +- librenms_model: "GLC-T" + netbox_module_type: "SFP-1G-T" + description: "Cisco SFP 1000BASE-T copper" + +- librenms_model: "GLC-TE" + netbox_module_type: "SFP-1G-T" + description: "Cisco SFP 1000BASE-T copper (extended temp)" + +- librenms_model: "SPP5200LR-C5" + netbox_module_type: "SFP-10G-LR" + description: "Cisco-branded Sumitomo SFP+ 10G-LR" + +- librenms_model: "SPP5310LR-C5" + netbox_module_type: "SFP-10G-LR" + description: "Cisco-branded Sumitomo SFP+ 10G-LR" + +- librenms_model: "SFBR-709SMZ-CS1" + netbox_module_type: "SFP-10G-SR" + description: "Cisco-branded Avago/Broadcom SFP+ 10G-SR" + +- librenms_model: "DP04QSDD-HE0" + netbox_module_type: "QSFP-DD-400G-ZR+" + description: "Cisco/Acacia QSFP-DD 400G-ZR+ coherent" + +- librenms_model: "QDD-400G-ZRP-S" + netbox_module_type: "QSFP-DD-400G-ZR+" + description: "Cisco QSFP-DD 400G-ZR+" + +- librenms_model: "QDD-400G-ZR4-S" + netbox_module_type: "QSFP-DD-400G-ZR" + description: "Cisco QSFP-DD 400G-ZR" + +# ─── Transceiver Mappings: Ciena ───────────────────────────────────────────── + +- librenms_model: "180-3530-900" + netbox_module_type: "QSFP-DD-400G-ZR" + description: "Ciena WaveLogic 5 Nano QSFP-DD 400ZR" + +- librenms_model: "176-3360-900" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Ciena QSFP-DD 400G-ZR-M coherent" + +- librenms_model: "176-3530-901" + netbox_module_type: "QSFP-DD-400G-ZR" + description: "Ciena QSFP-DD 400G-ZR coherent" + +- librenms_model: "176-3590-900" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Ciena QSFP-DD 400G-ZR-M coherent" + +# ─── Transceiver Mappings: T1 Nexus ───────────────────────────────────────── + +- librenms_model: "T1-QDD-400G-LR4" + netbox_module_type: "QSFP-DD-400G-LR4" + description: "T1 Nexus QSFP-DD 400G-LR4" + +- librenms_model: "T1-QDD-400G-FR4" + netbox_module_type: "QSFP-DD-400G-FR4" + description: "T1 Nexus QSFP-DD 400G-FR4" + +- librenms_model: "T1-QSFP28-LR4" + netbox_module_type: "QSFP-100G-LR4" + description: "T1 Nexus QSFP28 100G-LR4" + +- librenms_model: "100G-LR4_A3" + netbox_module_type: "QSFP-100G-LR4" + description: "T1 Nexus QSFP28 100G-LR4 (rev A3)" + +# ─── Transceiver Mappings: Innolight ──────────────────────────────────────── + +- librenms_model: "T-DQ4CNT-NCN" + netbox_module_type: "QSFP-DD-400G-FR4" + description: "Innolight QSFP-DD 400G-FR4" + +# ─── Transceiver Mappings: FS.com ──────────────────────────────────────────── + +- librenms_model: "Q28-PC03" + netbox_module_type: "QSFP28-100G-CU3M" + description: "FS.com QSFP28 100G passive DAC 3m" + +# ─── Transceiver Mappings: ProLabs ─────────────────────────────────────────── + +- librenms_model: "Q28LR431-10-IN" + netbox_module_type: "QSFP-100G-LR4" + description: "ProLabs QSFP28 100G-LR4 10km" + +# ─── Transceiver Mappings: Arcos Fixed-Port Part Numbers ───────────────────── + +- librenms_model: "SP7041-TE" + netbox_module_type: "SFP-1G-T" + description: "SFP 1000BASE-T copper (Arcos platform)" + +# ─── Transceiver Mappings: LeGrand Innolight ───────────────────────────────── + +- librenms_model: "LGI-FTLC9152RGPL" + netbox_module_type: "QSFP-100G-SWDM4" + description: "LeGrand-branded Finisar QSFP28 100G-SWDM4" + +# ─── Transceiver Mappings: Additional Finisar Variants ────────────────────── +# Some transceivers have customer-code suffixes that normalization may not handle. +# Add direct mappings as fallback. + +- librenms_model: "FTLC1151RDPL-CN" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4 (CN customer code)" + +- librenms_model: "FTLC1154RDPL-A5" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4 (A5 customer code)" + +# ─── Unknown / Unidentified Part Numbers ───────────────────────────────────── +# These are mapped based on port context (QSFP28 100G slot) when vendor is unknown. + +- librenms_model: "1F3QAA" + netbox_module_type: "QSFP-100G-LR4" + description: "Unknown QSFP28 100G (mapped by port context)" diff --git a/contrib/normalization_rules.yaml b/contrib/normalization_rules.yaml new file mode 100644 index 0000000000..c3d2081bea --- /dev/null +++ b/contrib/normalization_rules.yaml @@ -0,0 +1,61 @@ +# Normalization Rules — Examples +# +# Regex-based string transformations applied before module type, device type, +# or module bay matching. Rules run in priority order (lower first); each +# rule's output feeds the next. +# +# Import via: LibreNMS → Normalization Rules → Import → YAML +# +# Fields: +# scope — module_type, device_type, or module_bay +# manufacturer — Optional manufacturer name (must exist in NetBox). +# When set, the rule only fires for that manufacturer. +# match_pattern — Python regex (re.sub pattern) +# replacement — Replacement string (supports \1, \2 back-references) +# priority — Lower values run first (default 100) +# description — Optional note + +# ── Nokia revision suffix stripping ────────────────────────────────────────── +# Nokia ENTITY-MIB reports module/transceiver models with 4-char revision +# suffixes (e.g. 3HE16474AARA01). NetBox module types use the base part +# number (3HE16474AA). This rule strips the suffix before matching. +# +# Captures the 10-char base (3HE + 5 alnum + 2 quality-tier letters), +# discards the 2-letter revision code + 2-digit build number. +- scope: module_type + manufacturer: Nokia + match_pattern: "^(3HE\\w{5}[A-Z]{2})[A-Z]{2}\\d{2}$" + replacement: "\\1" + priority: 100 + description: "Strip Nokia revision suffixes (e.g. RA01, RB01, RG01) from ENTITY-MIB model strings" + +# ── Finisar / II-VI / Coherent suffix stripping ───────────────────────────── +# Finisar part numbers have customer-specific suffixes after a hyphen: +# FTLC1154RDPL-A5 (original Finisar) +# FTLC1154RDPL-C (Prolabs compatible) +# FTLX1474D3BCL-C1 (Cisco-coded Finisar) +# This rule strips everything after the last hyphen for FT... models. +- scope: module_type + match_pattern: "^(FT[A-Z0-9]+)-[A-Z0-9]+$" + replacement: "\\1" + priority: 100 + description: "Strip Finisar/II-VI customer suffixes (-A5, -C, -CN, -C1, etc.)" + +# ── Prolabs LGI- prefix stripping ─────────────────────────────────────────── +# Prolabs-compatible optics sometimes prepend LGI- to the OEM part number: +# LGI-FTLC9152RGPL → FTLC9152RGPL +- scope: module_type + match_pattern: "^LGI-(.+)$" + replacement: "\\1" + priority: 50 + description: "Strip Prolabs LGI- prefix from OEM part numbers" + +# ── Nokia transceiver model field cleanup ──────────────────────────────────── +# Nokia transceiver API sometimes returns model strings with trailing vendor +# info: "3HE10550AARA01 NOK IPU3BFUEAA" — extract just the part number. +- scope: module_type + manufacturer: Nokia + match_pattern: "^(3HE\\w+)\\s+.*$" + replacement: "\\1" + priority: 50 + description: "Extract Nokia part number from transceiver model field (strip trailing vendor/oui info)" diff --git a/docs/usage_tips/custom_field.md b/docs/usage_tips/custom_field.md index 032812ed82..7ed27a2f97 100644 --- a/docs/usage_tips/custom_field.md +++ b/docs/usage_tips/custom_field.md @@ -4,6 +4,9 @@ To enhance device identification and synchronization between NetBox and LibreNMS, this plugin supports using a custom field `librenms_id` on Device, Virtual Machine and Interface objects. While the plugin works without it, using this custom field is recommended for LibreNMS API lookups, and to assist with matching the remote device and remote interfaces for cable creation in Netbox. It can also be entered manually if no primary IP or FQDN is available. +!!! info "Automatic Creation" + As of version 0.4.2, the plugin **automatically creates** the `librenms_id` custom field when migrations are run. You no longer need to create it manually. The field is created for Device, Virtual Machine, Interface, and VM Interface objects. + For the Device and Virtual Machine objects the plugin will automatically populate the LibreNMS ID custom field when opening the LibreNMS Sync page if the device has been found in LibreNMS. For the Interface object, the plugin will automatically populate the LibreNMS ID custom field when the interface data is synced from LibreNMS. @@ -15,7 +18,10 @@ For the Interface object, the plugin will automatically populate the LibreNMS ID - **Efficient Synchronization:** Enhances the reliability of API lookups. - **Cable creation:** Allows better device identification for the creation of cables between NetBox devices. -## Suggested Custom Field Setup +## Manual Custom Field Setup (Legacy) + +!!! note + This section is only needed if you are running an older version of the plugin that does not auto-create the field, or if you need to recreate it after deletion. Follow these steps to create the `librenms_id` custom field in NetBox: diff --git a/docs/usage_tips/permissions.md b/docs/usage_tips/permissions.md index 9f9ecfb3f3..39c5225d2a 100644 --- a/docs/usage_tips/permissions.md +++ b/docs/usage_tips/permissions.md @@ -26,7 +26,7 @@ A user needs both tiers of permissions to complete an action. For example, to vi The Plugin also enforces Netbox object permissions so the following permission would also be required: -2. **Tier 2: Object permission**: User needs `dcim.add_device` (to create the device in NetBox) +1. **Tier 2: Object permission**: User needs `dcim.add_device` (to create the device in NetBox) If either permission is missing, the operation fails with an appropriate error message. diff --git a/netbox_librenms_plugin/__init__.py b/netbox_librenms_plugin/__init__.py index f1720c85d6..d0499c53f3 100644 --- a/netbox_librenms_plugin/__init__.py +++ b/netbox_librenms_plugin/__init__.py @@ -28,6 +28,7 @@ def ready(self): super().ready() from django.conf import settings + from django.db.models.signals import post_migrate plugin_config = getattr(settings, "PLUGINS_CONFIG", {}).get(self.name, {}) @@ -37,6 +38,12 @@ def ready(self): else: self._validate_legacy_config(plugin_config) + # Auto-create the librenms_id custom field after migrations complete + post_migrate.connect( + _ensure_librenms_id_custom_field, + dispatch_uid="netbox_librenms_plugin_ensure_cf", + ) + def _validate_multi_server_config(self, servers_config): """Validate multi-server configuration.""" if not servers_config or not isinstance(servers_config, dict): @@ -61,4 +68,61 @@ def _validate_legacy_config(self, plugin_config): ) +def _ensure_librenms_id_custom_field(sender, **kwargs): + """ + Auto-create the 'librenms_id' custom field if it doesn't exist. + Runs after migrations via post_migrate signal to ensure tables exist. + Uses dispatch_uid to avoid duplicate connections. + """ + # Only run once per migrate invocation (post_migrate fires per-app). + # The _executed flag is intentionally never reset: migrations are expected to + # run in short-lived CLI processes (manage.py migrate) where the flag is + # naturally cleared on exit. Long-running processes (e.g. gunicorn workers) + # should not rely on this handler re-executing after startup. + if getattr(_ensure_librenms_id_custom_field, "_executed", False): + return + _ensure_librenms_id_custom_field._executed = True # not reset; see comment above + + import logging + + try: + from django.contrib.contenttypes.models import ContentType + + from extras.models import CustomField + + cf, created = CustomField.objects.get_or_create( + name="librenms_id", + defaults={ + "type": "integer", + "label": "LibreNMS ID", + "description": "LibreNMS Device ID for synchronization (auto-created by plugin)", + "required": False, + "ui_visible": "if-set", + "ui_editable": "yes", + "is_cloneable": False, + }, + ) + + # Ensure the field is assigned to the required object types + from dcim.models import Device, Interface + from virtualization.models import VirtualMachine, VMInterface + + required_models = [Device, VirtualMachine, Interface, VMInterface] + current_types = set(cf.object_types.values_list("pk", flat=True)) + + for model in required_models: + ct = ContentType.objects.get_for_model(model) + if ct.pk not in current_types: + cf.object_types.add(ct) + + if created: + logging.getLogger("netbox_librenms_plugin").info( + "Auto-created 'librenms_id' custom field for Device, VirtualMachine, Interface, VMInterface" + ) + except Exception as e: + # Don't break startup if custom field creation fails (e.g., during initial migration), + # but log the error so it's not silently swallowed. + logging.getLogger("netbox_librenms_plugin").exception("Failed to auto-create 'librenms_id' custom field: %s", e) + + config = LibreNMSSyncConfig diff --git a/netbox_librenms_plugin/api/serializers.py b/netbox_librenms_plugin/api/serializers.py index 6bcd0aef20..bcde788d2b 100644 --- a/netbox_librenms_plugin/api/serializers.py +++ b/netbox_librenms_plugin/api/serializers.py @@ -1,6 +1,12 @@ from netbox.api.serializers import NetBoxModelSerializer -from netbox_librenms_plugin.models import InterfaceTypeMapping +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) class InterfaceTypeMappingSerializer(NetBoxModelSerializer): @@ -11,3 +17,51 @@ class Meta: model = InterfaceTypeMapping fields = ["id", "librenms_type", "librenms_speed", "netbox_type", "description"] + + +class DeviceTypeMappingSerializer(NetBoxModelSerializer): + """Serialize DeviceTypeMapping model for REST API.""" + + class Meta: + """Meta options for DeviceTypeMappingSerializer.""" + + model = DeviceTypeMapping + fields = ["id", "librenms_hardware", "netbox_device_type", "description"] + + +class ModuleTypeMappingSerializer(NetBoxModelSerializer): + """Serialize ModuleTypeMapping model for REST API.""" + + class Meta: + """Meta options for ModuleTypeMappingSerializer.""" + + model = ModuleTypeMapping + fields = ["id", "librenms_model", "netbox_module_type", "description"] + + +class ModuleBayMappingSerializer(NetBoxModelSerializer): + """Serialize ModuleBayMapping model for REST API.""" + + class Meta: + """Meta options for ModuleBayMappingSerializer.""" + + model = ModuleBayMapping + fields = ["id", "librenms_name", "librenms_class", "netbox_bay_name", "is_regex", "description"] + + +class NormalizationRuleSerializer(NetBoxModelSerializer): + """Serialize NormalizationRule model for REST API.""" + + class Meta: + """Meta options for NormalizationRuleSerializer.""" + + model = NormalizationRule + fields = [ + "id", + "scope", + "manufacturer", + "match_pattern", + "replacement", + "priority", + "description", + ] diff --git a/netbox_librenms_plugin/api/urls.py b/netbox_librenms_plugin/api/urls.py index 230aa078d0..c032e7b2f5 100644 --- a/netbox_librenms_plugin/api/urls.py +++ b/netbox_librenms_plugin/api/urls.py @@ -7,6 +7,10 @@ router = NetBoxRouter() router.register("interface-type-mappings", views.InterfaceTypeMappingViewSet) +router.register("device-type-mappings", views.DeviceTypeMappingViewSet) +router.register("module-type-mappings", views.ModuleTypeMappingViewSet) +router.register("module-bay-mappings", views.ModuleBayMappingViewSet) +router.register("normalization-rules", views.NormalizationRuleViewSet) urlpatterns = [ path("jobs//sync-status/", views.sync_job_status, name="sync_job_status"), diff --git a/netbox_librenms_plugin/api/views.py b/netbox_librenms_plugin/api/views.py index 768c67f5fe..287a3858c1 100644 --- a/netbox_librenms_plugin/api/views.py +++ b/netbox_librenms_plugin/api/views.py @@ -11,9 +11,21 @@ from rq.job import Job as RQJob from netbox_librenms_plugin.constants import PERM_CHANGE_PLUGIN, PERM_VIEW_PLUGIN -from netbox_librenms_plugin.models import InterfaceTypeMapping - -from .serializers import InterfaceTypeMappingSerializer +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) + +from .serializers import ( + DeviceTypeMappingSerializer, + InterfaceTypeMappingSerializer, + ModuleBayMappingSerializer, + ModuleTypeMappingSerializer, + NormalizationRuleSerializer, +) logger = logging.getLogger(__name__) @@ -22,8 +34,8 @@ class LibreNMSPluginPermission(BasePermission): """ Permission class for LibreNMS plugin API endpoints. - - GET requests require view_librenmssettings - - All other requests require change_librenmssettings + - Safe requests (GET, HEAD, OPTIONS) require netbox_librenms_plugin.view_librenmssettings + - All other requests require netbox_librenms_plugin.change_librenmssettings """ def has_permission(self, request, view): @@ -41,6 +53,42 @@ class InterfaceTypeMappingViewSet(NetBoxModelViewSet): serializer_class = InterfaceTypeMappingSerializer +class DeviceTypeMappingViewSet(NetBoxModelViewSet): + """API viewset for DeviceTypeMapping CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = DeviceTypeMapping.objects.all() + serializer_class = DeviceTypeMappingSerializer + + +class ModuleTypeMappingViewSet(NetBoxModelViewSet): + """API viewset for ModuleTypeMapping CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = ModuleTypeMapping.objects.all() + serializer_class = ModuleTypeMappingSerializer + + +class ModuleBayMappingViewSet(NetBoxModelViewSet): + """API viewset for ModuleBayMapping CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = ModuleBayMapping.objects.all() + serializer_class = ModuleBayMappingSerializer + + +class NormalizationRuleViewSet(NetBoxModelViewSet): + """API viewset for NormalizationRule CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = NormalizationRule.objects.all() + serializer_class = NormalizationRuleSerializer + + @api_view(["POST"]) @permission_classes([LibreNMSPluginPermission]) def sync_job_status(request, job_pk): diff --git a/netbox_librenms_plugin/filters.py b/netbox_librenms_plugin/filters.py index 9ec162a64c..134bd8962d 100644 --- a/netbox_librenms_plugin/filters.py +++ b/netbox_librenms_plugin/filters.py @@ -1,6 +1,6 @@ import django_filters -from .models import InterfaceTypeMapping +from .models import DeviceTypeMapping, InterfaceTypeMapping, ModuleBayMapping, ModuleTypeMapping, NormalizationRule class InterfaceTypeMappingFilterSet(django_filters.FilterSet): @@ -11,3 +11,43 @@ class Meta: model = InterfaceTypeMapping fields = ["librenms_type", "librenms_speed", "netbox_type", "description"] + + +class DeviceTypeMappingFilterSet(django_filters.FilterSet): + """Filter set for DeviceTypeMapping model.""" + + class Meta: + """Meta options for DeviceTypeMappingFilterSet.""" + + model = DeviceTypeMapping + fields = ["librenms_hardware", "description"] + + +class ModuleTypeMappingFilterSet(django_filters.FilterSet): + """Filter set for ModuleTypeMapping model.""" + + class Meta: + """Meta options for ModuleTypeMappingFilterSet.""" + + model = ModuleTypeMapping + fields = ["librenms_model", "description"] + + +class ModuleBayMappingFilterSet(django_filters.FilterSet): + """Filter set for ModuleBayMapping model.""" + + class Meta: + """Meta options for ModuleBayMappingFilterSet.""" + + model = ModuleBayMapping + fields = ["librenms_name", "librenms_class", "netbox_bay_name", "is_regex"] + + +class NormalizationRuleFilterSet(django_filters.FilterSet): + """Filter set for NormalizationRule model.""" + + class Meta: + """Meta options for NormalizationRuleFilterSet.""" + + model = NormalizationRule + fields = ["scope", "manufacturer"] diff --git a/netbox_librenms_plugin/forms.py b/netbox_librenms_plugin/forms.py index f3e3d075e5..62def6e786 100644 --- a/netbox_librenms_plugin/forms.py +++ b/netbox_librenms_plugin/forms.py @@ -2,7 +2,7 @@ import logging from dcim.choices import InterfaceTypeChoices -from dcim.models import Device, DeviceRole, DeviceType, Location, Rack, Site +from dcim.models import Device, DeviceRole, DeviceType, Location, Manufacturer, Rack, Site from django import forms from django.http import QueryDict from django.utils.translation import gettext_lazy as _ @@ -12,10 +12,22 @@ NetBoxModelImportForm, ) from netbox.plugins import get_plugin_config -from utilities.forms.fields import CSVChoiceField, DynamicModelMultipleChoiceField +from utilities.forms.fields import ( + CSVChoiceField, + CSVModelChoiceField, + DynamicModelChoiceField, + DynamicModelMultipleChoiceField, +) from virtualization.models import Cluster, VirtualMachine -from .models import InterfaceTypeMapping, LibreNMSSettings +from .models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + LibreNMSSettings, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) logger = logging.getLogger(__name__) @@ -51,11 +63,24 @@ def _get_librenms_poller_group_choices(): """ Helper function to get poller group choices from LibreNMS API. Shared between AddToLIbreSNMPV1V2 and AddToLIbreSNMPV3 forms. + Results are cached to avoid repeated API calls on every form instantiation. """ + from django.core.cache import cache + from .librenms_api import LibreNMSAPI choices = [("0", "Default (0)")] + try: + api = LibreNMSAPI() + server_id = api.librenms_url.rstrip("/") + cache_key = f"librenms_poller_group_choices_{server_id}" + except Exception: + cache_key = "librenms_poller_group_choices" + cached_choices = cache.get(cache_key) + if cached_choices: + return cached_choices + try: api = LibreNMSAPI() success, poller_groups = api.get_poller_groups() @@ -72,6 +97,8 @@ def _get_librenms_poller_group_choices(): else: label = f"{group_name} ({group_id})" choices.append((group_id, label)) + + cache.set(cache_key, choices, timeout=api.cache_timeout) except Exception: logger.exception("Failed to fetch LibreNMS poller groups; using default choices") @@ -90,10 +117,13 @@ class ServerConfigForm(NetBoxModelForm): ) class Meta: + """Meta options for ServerConfigForm.""" + model = LibreNMSSettings fields = ["selected_server"] def __init__(self, *args, **kwargs): + """Initialize form and populate server choices.""" super().__init__(*args, **kwargs) self.fields["selected_server"].choices = _get_librenms_server_choices() @@ -131,6 +161,8 @@ class ImportSettingsForm(NetBoxModelForm): ) class Meta: + """Meta options for ImportSettingsForm.""" + model = LibreNMSSettings fields = [ "vc_member_name_pattern", @@ -213,6 +245,8 @@ class InterfaceTypeMappingForm(NetBoxModelForm): """ class Meta: + """Meta options for InterfaceTypeMappingForm.""" + model = InterfaceTypeMapping fields = ["librenms_type", "librenms_speed", "netbox_type", "description"] @@ -230,6 +264,8 @@ class InterfaceTypeMappingImportForm(NetBoxModelImportForm): ) class Meta: + """Meta options for InterfaceTypeMappingImportForm.""" + model = InterfaceTypeMapping fields = ["librenms_type", "librenms_speed", "netbox_type", "description"] @@ -260,6 +296,163 @@ class InterfaceTypeMappingFilterForm(NetBoxModelFilterSetForm): model = InterfaceTypeMapping +class DeviceTypeMappingForm(NetBoxModelForm): + """Form for creating and editing device type mappings between LibreNMS and NetBox.""" + + netbox_device_type = forms.ModelChoiceField( + queryset=DeviceType.objects.all(), + label="NetBox Device Type", + widget=forms.Select(attrs={"class": "form-select"}), + ) + + class Meta: + """Meta options for DeviceTypeMappingForm.""" + + model = DeviceTypeMapping + fields = ["librenms_hardware", "netbox_device_type", "description"] + + +class DeviceTypeMappingImportForm(NetBoxModelImportForm): + """Form for bulk importing device type mappings.""" + + class Meta: + """Meta options for DeviceTypeMappingImportForm.""" + + model = DeviceTypeMapping + fields = ["librenms_hardware", "netbox_device_type", "description"] + + +class DeviceTypeMappingFilterForm(NetBoxModelFilterSetForm): + """Form for filtering device type mappings.""" + + librenms_hardware = forms.CharField(required=False, label="LibreNMS Hardware") + description = forms.CharField( + required=False, + label="Description", + help_text="Filter by description (partial match)", + ) + + model = DeviceTypeMapping + + +class ModuleTypeMappingForm(NetBoxModelForm): + """Form for creating and editing module type mappings between LibreNMS and NetBox.""" + + class Meta: + """Meta options for ModuleTypeMappingForm.""" + + model = ModuleTypeMapping + fields = ["librenms_model", "netbox_module_type", "description"] + + +class ModuleTypeMappingImportForm(NetBoxModelImportForm): + """Form for bulk importing module type mappings.""" + + class Meta: + """Meta options for ModuleTypeMappingImportForm.""" + + model = ModuleTypeMapping + fields = ["librenms_model", "netbox_module_type", "description"] + + +class ModuleTypeMappingFilterForm(NetBoxModelFilterSetForm): + """Form for filtering module type mappings.""" + + librenms_model = forms.CharField(required=False, label="LibreNMS Model") + description = forms.CharField( + required=False, + label="Description", + help_text="Filter by description (partial match)", + ) + + model = ModuleTypeMapping + + +class ModuleBayMappingForm(NetBoxModelForm): + """Form for creating and editing module bay mappings between LibreNMS and NetBox.""" + + class Meta: + """Meta options for ModuleBayMappingForm.""" + + model = ModuleBayMapping + fields = ["librenms_name", "librenms_class", "netbox_bay_name", "is_regex", "description"] + + +class ModuleBayMappingImportForm(NetBoxModelImportForm): + """Form for bulk importing module bay mappings.""" + + class Meta: + """Meta options for ModuleBayMappingImportForm.""" + + model = ModuleBayMapping + fields = ["librenms_name", "librenms_class", "netbox_bay_name", "is_regex", "description"] + + +class ModuleBayMappingFilterForm(NetBoxModelFilterSetForm): + """Form for filtering module bay mappings.""" + + librenms_name = forms.CharField(required=False, label="LibreNMS Name") + librenms_class = forms.CharField(required=False, label="LibreNMS Class") + netbox_bay_name = forms.CharField(required=False, label="NetBox Bay Name") + is_regex = forms.NullBooleanField(required=False, label="Regex") + + model = ModuleBayMapping + + +class NormalizationRuleForm(NetBoxModelForm): + """Form for creating and editing normalization rules.""" + + manufacturer = DynamicModelChoiceField( + queryset=Manufacturer.objects.all(), + required=False, + help_text="Optional: scope this rule to a specific manufacturer", + ) + + class Meta: + """Meta options for NormalizationRuleForm.""" + + model = NormalizationRule + fields = ["scope", "manufacturer", "match_pattern", "replacement", "priority", "description"] + + +class NormalizationRuleImportForm(NetBoxModelImportForm): + """Form for bulk importing normalization rules.""" + + scope = CSVChoiceField( + choices=NormalizationRule.SCOPE_CHOICES, + help_text="Scope: module_type, device_type, or module_bay", + ) + manufacturer = CSVModelChoiceField( + queryset=Manufacturer.objects.all(), + to_field_name="name", + required=False, + help_text="Optional manufacturer name (must already exist in NetBox)", + ) + + class Meta: + """Meta options for NormalizationRuleImportForm.""" + + model = NormalizationRule + fields = ["scope", "manufacturer", "match_pattern", "replacement", "priority", "description"] + + +class NormalizationRuleFilterForm(NetBoxModelFilterSetForm): + """Form for filtering normalization rules.""" + + scope = forms.ChoiceField( + required=False, + choices=[("", "---------")] + NormalizationRule.SCOPE_CHOICES, + label="Scope", + ) + manufacturer_id = DynamicModelChoiceField( + queryset=Manufacturer.objects.all(), + required=False, + label="Manufacturer", + ) + + model = NormalizationRule + + class AddToLIbreSNMPV1V2(forms.Form): """ Form for adding devices to LibreNMS using SNMPv1 or SNMPv2c authentication. @@ -315,6 +508,7 @@ class AddToLIbreSNMPV1V2(forms.Form): ) def __init__(self, *args, **kwargs): + """Initialize form and populate poller group choices.""" super().__init__(*args, **kwargs) self.fields["poller_group"].choices = _get_librenms_poller_group_choices() @@ -412,6 +606,7 @@ class AddToLIbreSNMPV3(forms.Form): ) def __init__(self, *args, **kwargs): + """Initialize form and populate poller group choices.""" super().__init__(*args, **kwargs) self.fields["poller_group"].choices = _get_librenms_poller_group_choices() @@ -422,6 +617,7 @@ class DeviceStatusFilterForm(NetBoxModelFilterSetForm): """ def __init__(self, *args, **kwargs): + """Initialize form and remove saved filter field.""" super().__init__(*args, **kwargs) # Remove the saved filter field if it exists if "filter_id" in self.fields: @@ -581,7 +777,9 @@ def _populate_librenms_locations(self): try: # Use caching to avoid repeated API calls - cache_key = "librenms_locations_choices" + api = LibreNMSAPI() + server_id = api.librenms_url.rstrip("/") + cache_key = f"librenms_locations_choices:{server_id}" cached_choices = cache.get(cache_key) if cached_choices: @@ -589,7 +787,6 @@ def _populate_librenms_locations(self): return # Fetch locations from LibreNMS - api = LibreNMSAPI() success, locations = api.get_locations() if success and locations: diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 0409b35da3..f910425226 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -7,6 +7,7 @@ from django.core.cache import cache from ..librenms_api import LibreNMSAPI +from ..utils import find_by_librenms_id from .cache import get_cache_metadata_key, get_import_device_cache_key, get_validated_device_cache_key from .device_operations import import_single_device, validate_device_for_import from .filters import get_librenms_devices_for_import @@ -267,45 +268,87 @@ def bulk_import_devices( ) -def _refresh_existing_device(validation: dict) -> None: - """Refresh existing_device from DB to pick up changes made in NetBox since caching.""" +def _refresh_existing_device(validation: dict, libre_device: dict = None) -> None: + """Refresh existing_device from DB to pick up changes made in NetBox since caching. + + When existing_device is None (wasn't found at cache time), re-check if the device + was imported since caching by looking up librenms_id or hostname. + """ existing = validation.get("existing_device") - if not existing or not hasattr(existing, "pk"): + if existing and hasattr(existing, "pk"): + try: + from dcim.models import Device + from virtualization.models import VirtualMachine + + if validation.get("import_as_vm"): + refreshed = VirtualMachine.objects.filter(pk=existing.pk).first() + else: + refreshed = Device.objects.filter(pk=existing.pk).first() + + if refreshed: + validation["existing_device"] = refreshed + if hasattr(refreshed, "role") and refreshed.role: + validation["device_role"] = {"found": True, "role": refreshed.role} + else: + # Device was deleted since caching — recompute readiness + validation["existing_device"] = None + validation["existing_match_type"] = None + validation["can_import"] = True + if validation.get("import_as_vm"): + validation["is_ready"] = bool( + validation.get("site", {}).get("found") and validation.get("device_role", {}).get("found") + ) + else: + validation["is_ready"] = bool( + validation.get("site", {}).get("found") + and validation.get("device_type", {}).get("found") + and validation.get("device_role", {}).get("found") + ) + except Exception as e: + existing_id = getattr(existing, "pk", "unknown") if existing else "none" + logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") + return + + # existing_device was None at cache time — check if device was imported since + if not libre_device: return try: from dcim.models import Device - from virtualization.models import VirtualMachine - if validation.get("import_as_vm"): - refreshed = VirtualMachine.objects.filter(pk=existing.pk).first() - else: - refreshed = Device.objects.filter(pk=existing.pk).first() - - if refreshed: - validation["existing_device"] = refreshed - if hasattr(refreshed, "role") and refreshed.role: - validation["device_role"]["found"] = True - validation["device_role"]["role"] = refreshed.role - else: - # Device was deleted since caching — recompute readiness - validation["existing_device"] = None - validation["existing_match_type"] = None - if validation.get("import_as_vm"): - required_found = ( - validation.get("site", {}).get("found") - and validation.get("cluster", {}).get("found") - and validation.get("device_role", {}).get("found") - ) - else: - required_found = ( - validation.get("site", {}).get("found") - and validation.get("device_type", {}).get("found") - and validation.get("device_role", {}).get("found") - ) - validation["can_import"] = validation["is_ready"] = bool(required_found and not validation.get("issues")) + librenms_id = libre_device.get("device_id") + hostname = libre_device.get("hostname", "") + sys_name = libre_device.get("sysName", "") + server_key = libre_device.get("_server_key") + + new_device = None + match_type = None + + # Check by librenms_id custom field first (JSON multi-server format + legacy) + if librenms_id: + try: + new_device = find_by_librenms_id(Device, int(librenms_id), server_key) + if new_device: + match_type = "librenms_id" + except (ValueError, TypeError): + pass + + # Fall back to hostname match + if not new_device and hostname: + new_device = Device.objects.filter(name__iexact=hostname).first() + if not new_device and sys_name: + new_device = Device.objects.filter(name__iexact=sys_name).first() + if new_device: + match_type = "hostname" + + if new_device: + validation["existing_device"] = new_device + validation["existing_match_type"] = match_type + validation["can_import"] = False + validation["is_ready"] = False + if hasattr(new_device, "role") and new_device.role: + validation["device_role"] = {"found": True, "role": new_device.role} except Exception as e: - existing_id = getattr(existing, "pk", "unknown") if existing else "none" - logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") + logger.error(f"Failed to check for newly imported device: {e}") def process_device_filters( @@ -470,7 +513,7 @@ def process_device_filters( # Refresh existing_device from DB to avoid stale data # (user may have changed role, name, etc. in NetBox) - _refresh_existing_device(device["_validation"]) + _refresh_existing_device(device["_validation"], libre_device=device) # Apply exclude_existing filter if enabled if exclude_existing: diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index c21d0bfb7a..8a7c37527d 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -20,6 +20,44 @@ logger = logging.getLogger(__name__) +def _try_chassis_device_type_match(api, device_id): + """ + Attempt device type matching using chassis inventory fields. + + When the LibreNMS hardware string doesn't match any NetBox device type, + the chassis entity often contains a more standardized identifier + (e.g., entPhysicalName 'CHAS-BP-MX480-S' or entPhysicalModelName '710-017414') + that matches a DeviceType part_number or model. + + Tries entPhysicalName first (typically the chassis part number), + then entPhysicalModelName as fallback. + + Returns: + dict with matched/device_type/match_type keys, or None on failure. + """ + skip_values = {"", "-", "Unspecified", "BUILTIN", "None"} + + try: + success, inventory = api.get_inventory_filtered(device_id, ent_physical_class="chassis") + if not success or not inventory: + return None + + for item in inventory: + # Try entPhysicalName first (often the chassis part number like CHAS-BP-MX480-S) + for field in ("entPhysicalName", "entPhysicalModelName"): + value = item.get(field) or "" + if value and value not in skip_values: + chassis_match = match_librenms_hardware_to_device_type(value) + if chassis_match["matched"]: + chassis_match["match_type"] = "chassis" + chassis_match["chassis_model"] = value + return chassis_match + except Exception: + logger.debug(f"Chassis inventory fallback failed for device {device_id}", exc_info=True) + + return None + + def _determine_device_name( libre_device: dict, use_sysname: bool = True, @@ -426,9 +464,20 @@ def validate_device_for_import( # 3. Validate DeviceType (required) hardware = libre_device.get("hardware", "") dt_match = match_librenms_hardware_to_device_type(hardware) + + # Chassis inventory fallback: when hardware doesn't match, + # try the chassis entPhysicalModelName as an additional lookup source + if not dt_match["matched"] and api: + device_id = libre_device.get("device_id") + if device_id: + chassis_match = _try_chassis_device_type_match(api, device_id) + if chassis_match and chassis_match["matched"]: + dt_match = chassis_match + result["device_type"] = dt_match if not dt_match["matched"]: + result["device_type"]["found"] = False result["issues"].append(f"No matching device type found for hardware: '{hardware}'") # Get some device types for user to choose from all_device_types = DeviceType.objects.all()[:10] diff --git a/netbox_librenms_plugin/librenms_api.py b/netbox_librenms_plugin/librenms_api.py index 8e2b2b3ac0..3291aad0cc 100644 --- a/netbox_librenms_plugin/librenms_api.py +++ b/netbox_librenms_plugin/librenms_api.py @@ -690,6 +690,51 @@ def get_device_inventory(self, device_id): except requests.exceptions.RequestException as e: return False, str(e) + def get_device_transceivers(self, device_id): + """ + Fetch all transceiver data for a device from LibreNMS. + + Route: /api/v0/devices/{device_id}/transceivers + + This is a separate data source from entity inventory. Some vendors + (e.g., Nokia/SROS) don't expose SFPs via ENTITY-MIB but do report + them through vendor-specific MIBs which LibreNMS surfaces here. + + Args: + device_id: LibreNMS device ID + + Returns: + tuple: (success: bool, data: list) + + Example transceiver item: + { + "port_id": 519, + "entity_physical_index": 1610899520, + "type": "CFP2/QSFP28", + "model": "3HE10550AARA01", + "serial": "X42AU0D", + "channels": 4, + "connector": "LC", + "wavelength": 1301, + ... + } + """ + try: + response = requests.get( + f"{self.librenms_url}/api/v0/devices/{device_id}/transceivers", + headers=self.headers, + timeout=DEFAULT_API_TIMEOUT, + verify=self.verify_ssl, + ) + response.raise_for_status() + + if response.status_code == 200: + data = response.json() + return True, data.get("transceivers", []) + return False, [] + except requests.exceptions.RequestException as e: + return False, str(e) + def get_poller_groups(self): """ Fetch all poller groups from LibreNMS. diff --git a/netbox_librenms_plugin/migrations/0009_add_devicetypemapping.py b/netbox_librenms_plugin/migrations/0009_add_devicetypemapping.py new file mode 100644 index 0000000000..dcd8fc4fd5 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0009_add_devicetypemapping.py @@ -0,0 +1,45 @@ +# Generated by Django 5.2.10 on 2026-02-17 11:48 + +import django.db.models.deletion +import netbox.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("dcim", "0225_gfk_indexes"), + ("extras", "0134_owner"), + ("netbox_librenms_plugin", "0008_librenmssettings_import_defaults"), + ] + + operations = [ + migrations.CreateModel( + name="DeviceTypeMapping", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ("librenms_hardware", models.CharField(max_length=255, unique=True)), + ("description", models.TextField(blank=True)), + ( + "netbox_device_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="librenms_mappings", + to="dcim.devicetype", + ), + ), + ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), + ], + options={ + "ordering": ["librenms_hardware"], + }, + bases=(netbox.models.deletion.DeleteMixin, models.Model), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0010_add_moduletypemapping.py b/netbox_librenms_plugin/migrations/0010_add_moduletypemapping.py new file mode 100644 index 0000000000..796bbceafd --- /dev/null +++ b/netbox_librenms_plugin/migrations/0010_add_moduletypemapping.py @@ -0,0 +1,49 @@ +# Generated by Django 5.2.10 on 2026-02-17 12:23 + +import django.db.models.deletion +import netbox.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("dcim", "0225_gfk_indexes"), + ("extras", "0134_owner"), + ("netbox_librenms_plugin", "0009_add_devicetypemapping"), + ] + + operations = [ + migrations.AlterModelOptions( + name="interfacetypemapping", + options={"ordering": ["librenms_type", "librenms_speed"]}, + ), + migrations.CreateModel( + name="ModuleTypeMapping", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ("librenms_model", models.CharField(max_length=255, unique=True)), + ("description", models.TextField(blank=True)), + ( + "netbox_module_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="librenms_mappings", + to="dcim.moduletype", + ), + ), + ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), + ], + options={ + "ordering": ["librenms_model"], + }, + bases=(netbox.models.deletion.DeleteMixin, models.Model), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0011_modulebaymapping.py b/netbox_librenms_plugin/migrations/0011_modulebaymapping.py new file mode 100644 index 0000000000..5b3c2c3be0 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0011_modulebaymapping.py @@ -0,0 +1,38 @@ +# Generated by Django 5.2.10 on 2026-02-17 12:29 + +import netbox.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("extras", "0134_owner"), + ("netbox_librenms_plugin", "0010_add_moduletypemapping"), + ] + + operations = [ + migrations.CreateModel( + name="ModuleBayMapping", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ("librenms_name", models.CharField(max_length=255)), + ("librenms_class", models.CharField(blank=True, max_length=50)), + ("netbox_bay_name", models.CharField(max_length=255)), + ("description", models.TextField(blank=True)), + ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), + ], + options={ + "ordering": ["librenms_name"], + "unique_together": {("librenms_name", "librenms_class")}, + }, + bases=(netbox.models.deletion.DeleteMixin, models.Model), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0012_add_is_regex_to_modulebaymapping.py b/netbox_librenms_plugin/migrations/0012_add_is_regex_to_modulebaymapping.py new file mode 100644 index 0000000000..52ff053e20 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0012_add_is_regex_to_modulebaymapping.py @@ -0,0 +1,18 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("netbox_librenms_plugin", "0011_modulebaymapping"), + ] + + operations = [ + migrations.AddField( + model_name="modulebaymapping", + name="is_regex", + field=models.BooleanField( + default=False, + help_text="Treat LibreNMS Name as a regex pattern with backreferences in NetBox Bay Name", + ), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0013_normalizationrule.py b/netbox_librenms_plugin/migrations/0013_normalizationrule.py new file mode 100644 index 0000000000..71d1f80509 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0013_normalizationrule.py @@ -0,0 +1,93 @@ +"""Restore NormalizationRule model. + +The table was created by earlier migrations (0013 + 0014 in a previous branch) +and already exists in the database. This migration uses SeparateDatabaseAndState +so Django's ORM knows about the model without trying to CREATE the table again. +If the table doesn't exist (fresh install), the database_operations handle creation. +""" + +import django.db.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("dcim", "0001_initial"), + ("extras", "0001_initial"), + ("netbox_librenms_plugin", "0012_add_is_regex_to_modulebaymapping"), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.CreateModel( + name="NormalizationRule", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ( + "scope", + models.CharField( + choices=[ + ("module_type", "Module Type"), + ("device_type", "Device Type"), + ("module_bay", "Module Bay"), + ], + max_length=50, + ), + ), + ("match_pattern", models.CharField(max_length=500)), + ("replacement", models.CharField(max_length=500)), + ("priority", models.PositiveIntegerField(default=100)), + ("description", models.TextField(blank=True)), + ( + "manufacturer", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="normalization_rules", + to="dcim.manufacturer", + ), + ), + ( + "tags", + taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag"), + ), + ], + options={ + "ordering": ["scope", "priority", "pk"], + }, + ), + ], + database_operations=[ + migrations.RunSQL( + sql=""" + CREATE TABLE IF NOT EXISTS "netbox_librenms_plugin_normalizationrule" ( + "id" bigserial NOT NULL PRIMARY KEY, + "created" timestamp with time zone NULL, + "last_updated" timestamp with time zone NULL, + "custom_field_data" jsonb NOT NULL DEFAULT '{}'::jsonb, + "scope" varchar(50) NOT NULL, + "match_pattern" varchar(500) NOT NULL, + "replacement" varchar(500) NOT NULL, + "priority" integer NOT NULL DEFAULT 100 CHECK ("priority" >= 0), + "description" text NOT NULL DEFAULT '', + "manufacturer_id" bigint NULL REFERENCES "dcim_manufacturer" ("id") + DEFERRABLE INITIALLY DEFERRED + ); + CREATE INDEX IF NOT EXISTS "netbox_librenms_plugin_norm_mfg_idx" + ON "netbox_librenms_plugin_normalizationrule" ("manufacturer_id"); + """, + reverse_sql="DROP TABLE IF EXISTS netbox_librenms_plugin_normalizationrule;", + ), + ], + ), + ] diff --git a/netbox_librenms_plugin/models.py b/netbox_librenms_plugin/models.py index cd79f47550..3978d9c5f4 100644 --- a/netbox_librenms_plugin/models.py +++ b/netbox_librenms_plugin/models.py @@ -1,4 +1,8 @@ +import re + from dcim.choices import InterfaceTypeChoices +from dcim.models import DeviceType, Manufacturer, ModuleType +from django.core.exceptions import ValidationError from django.db import models from django.urls import reverse from netbox.models import NetBoxModel @@ -71,6 +75,205 @@ class Meta: """Meta options for InterfaceTypeMapping.""" unique_together = ["librenms_type", "librenms_speed"] + ordering = ["librenms_type", "librenms_speed"] def __str__(self): return f"{self.librenms_type} + {self.librenms_speed} -> {self.netbox_type}" + + +class DeviceTypeMapping(NetBoxModel): + """Map LibreNMS hardware strings to NetBox DeviceType objects.""" + + librenms_hardware = models.CharField( + max_length=255, + unique=True, + help_text="Hardware string as reported by LibreNMS (e.g., 'Juniper MX480 Internet Backbone Router')", + ) + netbox_device_type = models.ForeignKey( + DeviceType, + on_delete=models.CASCADE, + related_name="librenms_mappings", + help_text="The NetBox DeviceType this hardware string maps to", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this mapping", + ) + + def get_absolute_url(self): + """Return the URL for this mapping's detail page.""" + return reverse("plugins:netbox_librenms_plugin:devicetypemapping_detail", args=[self.pk]) + + class Meta: + """Meta options for DeviceTypeMapping.""" + + ordering = ["librenms_hardware"] + + def __str__(self): + return f"{self.librenms_hardware} -> {self.netbox_device_type}" + + +class ModuleTypeMapping(NetBoxModel): + """Map LibreNMS inventory model names to NetBox ModuleType objects.""" + + librenms_model = models.CharField( + max_length=255, + unique=True, + help_text="Model name from LibreNMS inventory (entPhysicalModelName)", + ) + netbox_module_type = models.ForeignKey( + ModuleType, + on_delete=models.CASCADE, + related_name="librenms_mappings", + help_text="The NetBox ModuleType this model name maps to", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this mapping", + ) + + def get_absolute_url(self): + """Return the URL for this mapping's detail page.""" + return reverse("plugins:netbox_librenms_plugin:moduletypemapping_detail", args=[self.pk]) + + class Meta: + """Meta options for ModuleTypeMapping.""" + + ordering = ["librenms_model"] + + def __str__(self): + return f"{self.librenms_model} -> {self.netbox_module_type}" + + +class ModuleBayMapping(NetBoxModel): + """Map LibreNMS inventory names to NetBox module bay names. + + Used when LibreNMS inventory names don't match NetBox bay names exactly. + For example: LibreNMS "Power Supply 1" → NetBox "PS1". + When is_regex is True, librenms_name is treated as a regex pattern and + netbox_bay_name can use backreferences (\\1, \\2, etc.). + Mappings are global (not scoped to device type or manufacturer). + """ + + librenms_name = models.CharField( + max_length=255, + help_text="Name from LibreNMS inventory (entPhysicalName). " + "When 'Use Regex' is enabled, this is a Python regex pattern.", + ) + librenms_class = models.CharField( + max_length=50, + blank=True, + help_text="Optional entPhysicalClass filter (e.g. 'powerSupply', 'fan', 'module')", + ) + netbox_bay_name = models.CharField( + max_length=255, + help_text="NetBox module bay name to match. With regex, supports backreferences (\\1, \\2, etc.).", + ) + is_regex = models.BooleanField( + default=False, + help_text="Treat LibreNMS Name as a regex pattern with backreferences in NetBox Bay Name", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this mapping", + ) + + def clean(self): + """Validate that regex patterns compile when is_regex is True.""" + super().clean() + if self.is_regex: + try: + re.compile(self.librenms_name) + except re.error as e: + raise ValidationError({"librenms_name": f"Invalid regex: {e}"}) + + def get_absolute_url(self): + """Return the URL for this mapping's detail page.""" + return reverse("plugins:netbox_librenms_plugin:modulebaymapping_detail", args=[self.pk]) + + class Meta: + """Meta options for ModuleBayMapping.""" + + unique_together = ["librenms_name", "librenms_class"] + ordering = ["librenms_name"] + + def __str__(self): + cls = f" [{self.librenms_class}]" if self.librenms_class else "" + return f"{self.librenms_name}{cls} -> {self.netbox_bay_name}" + + +class NormalizationRule(NetBoxModel): + """Regex-based string normalization applied before matching lookups. + + Generic building block: a single rule engine handles normalization + for module types, device types, module bays, and future scopes. + Rules are applied in priority order; each transforms the string + for the next rule in the chain. + + Example – strip Nokia revision suffixes: + scope: module_type + match_pattern: ^(3HE\\w{5}[A-Z]{2})[A-Z]{2}\\d{2}$ + replacement: \\1 + Result: 3HE16474AARA01 → 3HE16474AA + """ + + SCOPE_MODULE_TYPE = "module_type" + SCOPE_DEVICE_TYPE = "device_type" + SCOPE_MODULE_BAY = "module_bay" + + SCOPE_CHOICES = [ + (SCOPE_MODULE_TYPE, "Module Type"), + (SCOPE_DEVICE_TYPE, "Device Type"), + (SCOPE_MODULE_BAY, "Module Bay"), + ] + + scope = models.CharField( + max_length=50, + choices=SCOPE_CHOICES, + help_text="Which matching lookup this rule applies to", + ) + manufacturer = models.ForeignKey( + Manufacturer, + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="normalization_rules", + help_text="Optional: only apply this rule to items from this manufacturer. " + "Leave blank for vendor-agnostic rules.", + ) + match_pattern = models.CharField( + max_length=500, + help_text="Regex pattern to match against input string (Python re syntax)", + ) + replacement = models.CharField( + max_length=500, + help_text="Replacement string (supports regex back-references \\1, \\2, …)", + ) + priority = models.PositiveIntegerField( + default=100, + help_text="Lower values run first. Rules chain: each transforms the output of the previous.", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this rule", + ) + + def clean(self): + """Validate that match_pattern compiles as a regex.""" + super().clean() + try: + re.compile(self.match_pattern) + except re.error as e: + raise ValidationError({"match_pattern": f"Invalid regex: {e}"}) + + def get_absolute_url(self): + """Return the URL for this rule's detail page.""" + return reverse("plugins:netbox_librenms_plugin:normalizationrule_detail", args=[self.pk]) + + class Meta: + """Meta options for NormalizationRule.""" + + ordering = ["scope", "priority", "pk"] + + def __str__(self): + return f"[{self.get_scope_display()}] {self.match_pattern} → {self.replacement}" diff --git a/netbox_librenms_plugin/navigation.py b/netbox_librenms_plugin/navigation.py index a08e62740f..052c06363c 100644 --- a/netbox_librenms_plugin/navigation.py +++ b/netbox_librenms_plugin/navigation.py @@ -31,6 +31,74 @@ ), ), ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:devicetypemapping_list", + link_text="Device Type Mappings", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:devicetypemapping_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:devicetypemapping_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:moduletypemapping_list", + link_text="Module Type Mappings", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:moduletypemapping_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:moduletypemapping_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:modulebaymapping_list", + link_text="Module Bay Mappings", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:modulebaymapping_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:modulebaymapping_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:normalizationrule_list", + link_text="Normalization Rules", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:normalizationrule_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:normalizationrule_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), ), ), ( diff --git a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js index cd470af1b1..0b42112b24 100644 --- a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js +++ b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js @@ -153,11 +153,15 @@ function initializeCountdowns() { if (window.vlanCountdownInterval) { clearInterval(window.vlanCountdownInterval); } + if (window.moduleCountdownInterval) { + clearInterval(window.moduleCountdownInterval); + } window.interfaceCountdownInterval = initializeCountdown("countdown-timer"); window.cableCountdownInterval = initializeCountdown("cable-countdown-timer"); window.ipCountdownInterval = initializeCountdown("ip-countdown-timer"); window.vlanCountdownInterval = initializeCountdown("vlan-countdown-timer"); + window.moduleCountdownInterval = initializeCountdown("module-countdown-timer"); } // ============================================ diff --git a/netbox_librenms_plugin/tables/mappings.py b/netbox_librenms_plugin/tables/mappings.py index 73949fd2c8..4b4b31a41d 100644 --- a/netbox_librenms_plugin/tables/mappings.py +++ b/netbox_librenms_plugin/tables/mappings.py @@ -1,7 +1,13 @@ import django_tables2 as tables from netbox.tables import NetBoxTable, columns -from netbox_librenms_plugin.models import InterfaceTypeMapping +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) class InterfaceTypeMappingTable(NetBoxTable): @@ -36,3 +42,132 @@ class Meta: "actions", ) attrs = {"class": "table table-hover table-headings table-striped"} + + +class DeviceTypeMappingTable(NetBoxTable): + """Table for displaying DeviceTypeMapping data.""" + + librenms_hardware = tables.Column(verbose_name="LibreNMS Hardware", linkify=True) + netbox_device_type = tables.Column(verbose_name="NetBox Device Type", linkify=True) + description = tables.Column(verbose_name="Description", linkify=False) + actions = columns.ActionsColumn(actions=("edit", "delete")) + + class Meta: + """Meta options for DeviceTypeMappingTable.""" + + model = DeviceTypeMapping + fields = ( + "id", + "librenms_hardware", + "netbox_device_type", + "description", + "actions", + ) + default_columns = ( + "id", + "librenms_hardware", + "netbox_device_type", + "description", + "actions", + ) + attrs = {"class": "table table-hover table-headings table-striped"} + + +class ModuleTypeMappingTable(NetBoxTable): + """Table for displaying ModuleTypeMapping data.""" + + librenms_model = tables.Column(verbose_name="LibreNMS Model", linkify=True) + netbox_module_type = tables.Column(verbose_name="NetBox Module Type", linkify=True) + description = tables.Column(verbose_name="Description", linkify=False) + actions = columns.ActionsColumn(actions=("edit", "delete")) + + class Meta: + """Meta options for ModuleTypeMappingTable.""" + + model = ModuleTypeMapping + fields = ( + "id", + "librenms_model", + "netbox_module_type", + "description", + "actions", + ) + default_columns = ( + "id", + "librenms_model", + "netbox_module_type", + "description", + "actions", + ) + attrs = {"class": "table table-hover table-headings table-striped"} + + +class ModuleBayMappingTable(NetBoxTable): + """Table for displaying ModuleBayMapping data.""" + + librenms_name = tables.Column(verbose_name="LibreNMS Name", linkify=True) + librenms_class = tables.Column(verbose_name="LibreNMS Class") + netbox_bay_name = tables.Column(verbose_name="NetBox Bay Name") + is_regex = columns.BooleanColumn(verbose_name="Regex") + description = tables.Column(verbose_name="Description", linkify=False) + actions = columns.ActionsColumn(actions=("edit", "delete")) + + class Meta: + """Meta options for ModuleBayMappingTable.""" + + model = ModuleBayMapping + fields = ( + "id", + "librenms_name", + "librenms_class", + "netbox_bay_name", + "is_regex", + "description", + "actions", + ) + default_columns = ( + "id", + "librenms_name", + "librenms_class", + "netbox_bay_name", + "is_regex", + "description", + "actions", + ) + attrs = {"class": "table table-hover table-headings table-striped"} + + +class NormalizationRuleTable(NetBoxTable): + """Table for displaying NormalizationRule data.""" + + scope = tables.Column(verbose_name="Scope", linkify=True) + manufacturer = tables.Column(verbose_name="Manufacturer", linkify=True) + match_pattern = tables.Column(verbose_name="Match Pattern") + replacement = tables.Column(verbose_name="Replacement") + priority = tables.Column(verbose_name="Priority") + description = tables.Column(verbose_name="Description", linkify=False) + actions = columns.ActionsColumn(actions=("edit", "delete")) + + class Meta: + """Meta options for NormalizationRuleTable.""" + + model = NormalizationRule + fields = ( + "id", + "scope", + "manufacturer", + "match_pattern", + "replacement", + "priority", + "description", + "actions", + ) + default_columns = ( + "id", + "scope", + "match_pattern", + "replacement", + "priority", + "actions", + ) + attrs = {"class": "table table-hover table-headings table-striped"} diff --git a/netbox_librenms_plugin/tables/modules.py b/netbox_librenms_plugin/tables/modules.py new file mode 100644 index 0000000000..458364fe10 --- /dev/null +++ b/netbox_librenms_plugin/tables/modules.py @@ -0,0 +1,180 @@ +import django_tables2 as tables +from django.urls import reverse +from django.utils.html import format_html +from utilities.paginator import EnhancedPaginator + +from netbox_librenms_plugin.utils import get_table_paginate_count + + +class LibreNMSModuleTable(tables.Table): + """Table for displaying LibreNMS inventory items mapped to NetBox modules.""" + + name = tables.Column(verbose_name="Name", attrs={"td": {"data-col": "name"}}) + model = tables.Column(verbose_name="Model", attrs={"td": {"data-col": "model"}}) + serial = tables.Column(verbose_name="Serial", attrs={"td": {"data-col": "serial"}}) + description = tables.Column(verbose_name="Description", attrs={"td": {"data-col": "description"}}) + item_class = tables.Column(verbose_name="Class", attrs={"td": {"data-col": "item_class"}}) + module_bay = tables.Column(verbose_name="Module Bay", attrs={"td": {"data-col": "module_bay"}}) + module_type = tables.Column(verbose_name="Module Type", attrs={"td": {"data-col": "module_type"}}) + status = tables.Column(verbose_name="Status", attrs={"td": {"data-col": "status"}}) + actions = tables.Column( + verbose_name="Actions", orderable=False, empty_values=(), attrs={"td": {"data-col": "actions"}} + ) + + class Meta: + attrs = {"class": "table table-hover object-list", "id": "librenms-module-table"} + row_attrs = {"class": lambda record: record.get("row_class", "")} + + def __init__(self, *args, device=None, **kwargs): + """Initialize table with optional device context.""" + self.device = device + self.csrf_token = "" + super().__init__(*args, **kwargs) + self.tab = "modules" + self.htmx_url = None + self.prefix = "modules_" + + def configure(self, request): + """Configure pagination settings and CSRF token.""" + from django.middleware.csrf import get_token + + self.csrf_token = get_token(request) + paginate = {"paginator_class": EnhancedPaginator, "per_page": get_table_paginate_count(request, self.prefix)} + tables.RequestConfig(request, paginate).configure(self) + + def render_name(self, value, record): + """Render inventory item name with tree indentation for sub-components.""" + depth = record.get("depth", 0) + if depth == 0: + return value or "-" + # Build visual tree prefix based on nesting depth + padding_px = depth * 20 + prefix = "└─ " + return format_html('{}{}', padding_px, prefix, value or "-") + + def render_model(self, value, record): + """Render model with link to module type if matched.""" + if not value or value == "-": + return "-" + if url := record.get("module_type_url"): + return format_html('{}', url, value) + return value + + def render_serial(self, value, record): + """Render serial number.""" + return value or "-" + + def render_description(self, value, record): + """Render description, truncated for display.""" + if not value: + return "-" + if len(value) > 60: + return format_html('{}…', value, value[:57]) + return value + + def render_item_class(self, value, record): + """Render the entPhysicalClass with an icon.""" + icons = { + "module": "mdi-expansion-card", + "ioModule": "mdi-expansion-card", + "cpmModule": "mdi-expansion-card", + "mdaModule": "mdi-expansion-card", + "fabricModule": "mdi-expansion-card", + "xioModule": "mdi-expansion-card", + "powerSupply": "mdi-power-plug", + "fan": "mdi-fan", + "port": "mdi-ethernet", + "other": "mdi-card-outline", + } + icon = icons.get(value, "mdi-card-outline") + return format_html(' {}', icon, value) + + def render_module_bay(self, value, record): + """Render module bay with link if found in NetBox.""" + if not value or value == "-": + return format_html('No matching bay') + if url := record.get("module_bay_url"): + return format_html('{}', url, value) + return value + + def render_module_type(self, value, record): + """Render module type match status.""" + if not value or value == "-": + return format_html('No matching type') + if url := record.get("module_type_url"): + return format_html('{}', url, value) + return value + + def render_status(self, value, record): + """Render sync status with badge.""" + badge_classes = { + "Installed": "bg-success", + "Matched": "bg-info", + "No Bay": "bg-warning", + "No Type": "bg-warning", + "Unmatched": "bg-secondary", + "Serial Mismatch": "bg-danger", + "Requires Upgrade": "bg-warning", + "Name Conflict": "bg-warning", + } + badge_class = badge_classes.get(value, "bg-secondary") + if warning := record.get("module_path_warning"): + return format_html('{}', badge_class, warning, value) + if warning := record.get("name_conflict_warning"): + return format_html( + '{}' + ' ', + badge_class, + warning, + value, + warning, + ) + return format_html('{}', badge_class, value) + + def render_actions(self, value, record): + """Render install button for matched modules and install branch for parents.""" + if not self.device: + return "" + + buttons = [] + + # Single install button + if record.get("can_install"): + url = reverse("plugins:netbox_librenms_plugin:install_module", kwargs={"pk": self.device.pk}) + buttons.append( + format_html( + '' + '' + '' + '' + '' + '", + url, + self.csrf_token, + record.get("module_bay_id", ""), + record.get("module_type_id", ""), + record.get("serial", ""), + ) + ) + + # Install branch button for parents with installable children + if record.get("has_installable_children") and record.get("ent_physical_index"): + url = reverse("plugins:netbox_librenms_plugin:install_branch", kwargs={"pk": self.device.pk}) + buttons.append( + format_html( + '
' + '' + '' + '
", + url, + self.csrf_token, + record.get("ent_physical_index", ""), + ) + ) + + return format_html("{}", format_html("".join(str(b) for b in buttons))) if buttons else "" diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html new file mode 100644 index 0000000000..ce6e430bfa --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html @@ -0,0 +1,31 @@ +{% load helpers %} +{% include 'inc/messages.html' %} + + +{% if module_sync.table %} +
+
+ + Showing inventory items from LibreNMS matched against NetBox module bays and module types. + +
+ {% if module_sync.cache_expiry %} +
+ Cache expires in: +
+ {% endif %} +
+ +
+ {% include 'netbox_librenms_plugin/inc/paginator.html' with table=module_sync.table %} + {% include 'inc/table.html' with table=module_sync.table %} + {% include 'netbox_librenms_plugin/inc/paginator.html' with table=module_sync.table %} +
+{% else %} +
+
+ +

No inventory data loaded. Click Refresh Modules to fetch data from LibreNMS.

+
+
+{% endif %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping.html new file mode 100644 index 0000000000..3894441179 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping.html @@ -0,0 +1,28 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} + +{% block content %} +
+
+
+ + + + + + + + + + + + + + + +
LibreNMS HardwareNetBox Device TypeDescription
{{ object.librenms_hardware }}{{ object.netbox_device_type }}{{ object.description|default:"—" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping_list.html new file mode 100644 index 0000000000..06c95270b3 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping_list.html @@ -0,0 +1,12 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Device Type Mapping

+

Map LibreNMS hardware strings to NetBox device types. + When importing devices from LibreNMS, these mappings are checked first before + falling back to exact part number / model matching.

+

Example: Map "Juniper MX480 Internet Backbone Router" to device type "MX480"

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html index 2bb7fb7f8c..f2f181da13 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html @@ -177,7 +177,7 @@
Device Information
{% endif %} {% elif validation.device_type.device_type %} - {{ validation.device_type.device_type }} + {{ validation.device_type.device_type }} {% else %} No matching type {% endif %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/inc/_module_sync.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/inc/_module_sync.html new file mode 100644 index 0000000000..4c2c5ae65d --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/inc/_module_sync.html @@ -0,0 +1,27 @@ +{% load helpers %} + + +
+

Module Sync

+
+
+ {% csrf_token %} + {% if has_librenms_id %} + {% with model_name=object|meta:"model_name" %} + {% if model_name == "device" %} + + {% endif %} + {% endwith %} + {% endif %} +
+
+
+ + +
+ {% include 'netbox_librenms_plugin/_module_sync_content.html' %} +
diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html index 82fd96ff03..4eb4b78106 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html @@ -249,7 +249,7 @@
Device Information Sync
{{ object.name }}
- {% if sysName and sysName != object.name %} + {% if sysName and sysName != "-" and sysName != object.name %}
@@ -259,7 +259,7 @@
Device Information Sync
Sync to NetBox
- {% elif sysName %} + {% elif sysName and sysName != "-" %} @@ -535,6 +535,14 @@
Device Information Sync
{% endif %} {% endwith %} + {% if module_sync %} + + {% endif %}
Device Information Sync {% include 'netbox_librenms_plugin/_ipaddress_sync.html' %}
+ {% if module_sync %} +
+ {% include 'netbox_librenms_plugin/inc/_module_sync.html' %} +
+ {% endif %} + {% with model_name=object|meta:"model_name" %} {% if model_name == "device" %}
+
+
+ + + + + + + + + + + + + + + + + +
LibreNMS NameLibreNMS ClassNetBox Bay NameDescription
{{ object.librenms_name }}{{ object.librenms_class|default:"—" }}{{ object.netbox_bay_name }}{{ object.description|default:"—" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping_list.html new file mode 100644 index 0000000000..fb87f901ec --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping_list.html @@ -0,0 +1,12 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Module Bay Mapping

+

Map LibreNMS inventory container names to NetBox module bay names. + When synchronizing modules from LibreNMS, these mappings determine which + NetBox module bay a LibreNMS component should be installed into.

+

Example: Map "Linecard(slot 1)" to "Slot 1", or "Power Supply 1" to "PS1"

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping.html new file mode 100644 index 0000000000..019b0e51ed --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping.html @@ -0,0 +1,28 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} + +{% block content %} +
+
+
+ + + + + + + + + + + + + + + +
LibreNMS ModelNetBox Module TypeDescription
{{ object.librenms_model }}{{ object.netbox_module_type }}{{ object.description|default:"—" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping_list.html new file mode 100644 index 0000000000..4cfc22d592 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping_list.html @@ -0,0 +1,12 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Module Type Mapping

+

Map LibreNMS inventory model names (entPhysicalModelName) to NetBox module types. + When synchronizing modules from LibreNMS, these mappings are checked first before + falling back to exact model / part number matching.

+

Example: Map "710-017414" to module type "WS-X4908-10GE"

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule.html new file mode 100644 index 0000000000..a1be7537a3 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule.html @@ -0,0 +1,34 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} + +{% block content %} +
+
+
+ + + + + + + + + + + + + + + + + + + + + +
ScopeManufacturerMatch PatternReplacementPriorityDescription
{{ object.get_scope_display }}{% if object.manufacturer %}{{ object.manufacturer }}{% else %}—{% endif %}{{ object.match_pattern }}{{ object.replacement }}{{ object.priority }}{{ object.description|default:"—" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule_list.html new file mode 100644 index 0000000000..d543141680 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule_list.html @@ -0,0 +1,16 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Normalization Rules

+

Regex-based string normalization applied before matching lookups. + When a LibreNMS string doesn't match any NetBox object or mapping entry, + normalization rules transform it (e.g. strip revision suffixes) and retry.

+

Rules are chained in priority order per scope. One rule engine serves + module types, device types, and module bays.

+

Example — strip Nokia revision suffixes:
+ ^(3HE\w{5}[A-Z]{2})[A-Z]{2}\d{2}$ → \1
+ Turns 3HE16474AARA01 into 3HE16474AA which matches the part number.

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/tests/test_init.py b/netbox_librenms_plugin/tests/test_init.py new file mode 100644 index 0000000000..6225484dd6 --- /dev/null +++ b/netbox_librenms_plugin/tests/test_init.py @@ -0,0 +1,171 @@ +"""Tests for netbox_librenms_plugin.__init__ module. + +Covers the _ensure_librenms_id_custom_field post_migrate signal handler. +""" + +from unittest.mock import MagicMock, patch + + +# ============================================================================= +# TestEnsureLibreNMSIdCustomField - 6 tests +# ============================================================================= + + +class TestEnsureLibreNMSIdCustomField: + """Test _ensure_librenms_id_custom_field signal handler.""" + + def setup_method(self): + """Reset the _executed flag before each test for consistent isolation.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + _ensure_librenms_id_custom_field._executed = False + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_creates_custom_field_when_missing( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """Custom field is created with correct defaults when it does not exist.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [] + MockCustomField.objects.get_or_create.return_value = (mock_cf, True) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + with patch("logging.getLogger") as mock_get_logger: + _ensure_librenms_id_custom_field(sender=None) + + MockCustomField.objects.get_or_create.assert_called_once_with( + name="librenms_id", + defaults={ + "type": "integer", + "label": "LibreNMS ID", + "description": "LibreNMS Device ID for synchronization (auto-created by plugin)", + "required": False, + "ui_visible": "if-set", + "ui_editable": "yes", + "is_cloneable": False, + }, + ) + + # Should have added content types for all 4 models + assert mock_cf.object_types.add.call_count == 4 + + # Should log when created + mock_get_logger.assert_called_with("netbox_librenms_plugin") + mock_get_logger.return_value.info.assert_called_once() + + def test_skips_when_already_executed(self): + """Handler is a no-op on second invocation (per-migrate dedup).""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + _ensure_librenms_id_custom_field._executed = True + + with patch("extras.models.CustomField") as MockCustomField: + _ensure_librenms_id_custom_field(sender=None) + MockCustomField.objects.get_or_create.assert_not_called() + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_existing_field_not_recreated( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """When custom field already exists, it is not recreated but types are checked.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2, 3, 4] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + _ensure_librenms_id_custom_field(sender=None) + + # All pks already present, no types should be added + mock_cf.object_types.add.assert_not_called() + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_adds_missing_content_types( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """When some content types are missing, only those are added.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + ct_existing = MagicMock() + ct_existing.pk = 1 + ct_new = MagicMock() + ct_new.pk = 99 + MockContentType.objects.get_for_model.side_effect = [ct_existing, ct_existing, ct_new, ct_new] + + _ensure_librenms_id_custom_field(sender=None) + + assert mock_cf.object_types.add.call_count == 2 + mock_cf.object_types.add.assert_any_call(ct_new) + + @patch("extras.models.CustomField") + def test_exception_does_not_propagate(self, MockCustomField): + """Exceptions during custom field creation are caught and logged.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + MockCustomField.objects.get_or_create.side_effect = Exception("DB not ready") + + with patch("logging.getLogger") as mock_get_logger: + # Should not raise + _ensure_librenms_id_custom_field(sender=None) + + # Verify the exception was logged + logger_instance = mock_get_logger.return_value + logger_instance.exception.assert_called_once() + call_args = logger_instance.exception.call_args + assert "librenms_id" in call_args[0][0] + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_no_log_when_field_already_exists( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """No log message when the custom field already existed.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2, 3, 4] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + with patch("logging.getLogger") as mock_get_logger: + _ensure_librenms_id_custom_field(sender=None) + # When the field already exists (created=False), the info log should + # not be emitted. We verify via the logger instance rather than + # asserting getLogger was never called, which is fragile. + logger_instance = mock_get_logger.return_value + logger_instance.info.assert_not_called() diff --git a/netbox_librenms_plugin/tests/test_utils.py b/netbox_librenms_plugin/tests/test_utils.py index 96065ab760..d98ca28b9c 100644 --- a/netbox_librenms_plugin/tests/test_utils.py +++ b/netbox_librenms_plugin/tests/test_utils.py @@ -15,9 +15,12 @@ class TestDeviceTypeMatching: """Test device type matching logic.""" + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") @patch("dcim.models.DeviceType") - def test_match_device_type_exact_match_by_part_number(self, mock_device_type): + def test_match_device_type_exact_match_by_part_number(self, mock_device_type, mock_mapping): """Exact part_number string should match.""" + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.side_effect = mock_mapping.DoesNotExist mock_dt = MagicMock(id=1, model="C9300-48P") mock_device_type.objects.get.return_value = mock_dt @@ -29,9 +32,12 @@ def test_match_device_type_exact_match_by_part_number(self, mock_device_type): assert result["device_type"] == mock_dt assert result["match_type"] == "exact" + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") @patch("dcim.models.DeviceType") - def test_match_device_type_exact_match_by_model(self, mock_device_type): + def test_match_device_type_exact_match_by_model(self, mock_device_type, mock_mapping): """Exact model string should match when part_number fails.""" + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.side_effect = mock_mapping.DoesNotExist mock_dt = MagicMock(id=1, model="WS-C3750X-48P") # Part number lookup fails, model lookup succeeds mock_device_type.DoesNotExist = Exception @@ -48,9 +54,12 @@ def test_match_device_type_exact_match_by_model(self, mock_device_type): assert result["device_type"] == mock_dt assert result["match_type"] == "exact" + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") @patch("dcim.models.DeviceType") - def test_match_device_type_not_found(self, mock_device_type): + def test_match_device_type_not_found(self, mock_device_type, mock_mapping): """Returns None when no match found.""" + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.side_effect = mock_mapping.DoesNotExist mock_device_type.DoesNotExist = Exception mock_device_type.objects.get.side_effect = mock_device_type.DoesNotExist @@ -62,6 +71,22 @@ def test_match_device_type_not_found(self, mock_device_type): assert result["device_type"] is None assert result["match_type"] is None + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") + def test_match_device_type_mapping_match(self, mock_mapping): + """DeviceTypeMapping entry should be used before part_number/model fallback.""" + mock_dt = MagicMock(id=1, model="MX480") + mock_mapping_obj = MagicMock(netbox_device_type=mock_dt) + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.return_value = mock_mapping_obj + + from netbox_librenms_plugin.utils import match_librenms_hardware_to_device_type + + result = match_librenms_hardware_to_device_type("Juniper MX480 Internet Backbone Router") + + assert result["matched"] is True + assert result["device_type"] == mock_dt + assert result["match_type"] == "mapping" + def test_match_device_type_empty_hardware(self): """Empty string returns None.""" from netbox_librenms_plugin.utils import match_librenms_hardware_to_device_type diff --git a/netbox_librenms_plugin/urls.py b/netbox_librenms_plugin/urls.py index 9eafdb1dca..2270905288 100644 --- a/netbox_librenms_plugin/urls.py +++ b/netbox_librenms_plugin/urls.py @@ -1,6 +1,6 @@ from django.urls import include, path -from .models import InterfaceTypeMapping +from .models import DeviceTypeMapping, InterfaceTypeMapping, ModuleBayMapping, ModuleTypeMapping, NormalizationRule from .views import ( AddDeviceToLibreNMSView, AssignVCSerialView, @@ -14,12 +14,23 @@ DeviceInterfaceTableView, DeviceIPAddressTableView, DeviceLibreNMSSyncView, + DeviceModuleTableView, DeviceRackUpdateView, DeviceRoleUpdateView, DeviceStatusListView, + DeviceTypeMappingBulkDeleteView, + DeviceTypeMappingBulkImportView, + DeviceTypeMappingChangeLogView, + DeviceTypeMappingCreateView, + DeviceTypeMappingDeleteView, + DeviceTypeMappingEditView, + DeviceTypeMappingListView, + DeviceTypeMappingView, DeviceValidationDetailsView, DeviceVCDetailsView, DeviceVLANTableView, + InstallBranchView, + InstallModuleView, InterfaceTypeMappingBulkDeleteView, InterfaceTypeMappingBulkImportView, InterfaceTypeMappingChangeLogView, @@ -30,6 +41,30 @@ InterfaceTypeMappingView, LibreNMSImportView, LibreNMSSettingsView, + ModuleBayMappingBulkDeleteView, + ModuleBayMappingBulkImportView, + ModuleBayMappingChangeLogView, + ModuleBayMappingCreateView, + ModuleBayMappingDeleteView, + ModuleBayMappingEditView, + ModuleBayMappingListView, + ModuleBayMappingView, + ModuleTypeMappingBulkDeleteView, + ModuleTypeMappingBulkImportView, + ModuleTypeMappingChangeLogView, + ModuleTypeMappingCreateView, + ModuleTypeMappingDeleteView, + ModuleTypeMappingEditView, + ModuleTypeMappingListView, + ModuleTypeMappingView, + NormalizationRuleBulkDeleteView, + NormalizationRuleBulkImportView, + NormalizationRuleChangeLogView, + NormalizationRuleCreateView, + NormalizationRuleDeleteView, + NormalizationRuleEditView, + NormalizationRuleListView, + NormalizationRuleView, SaveUserPrefView, SingleCableVerifyView, SingleInterfaceVerifyView, @@ -71,6 +106,21 @@ DeviceCableTableView.as_view(), name="device_cable_sync", ), + path( + "devices//module-sync/", + DeviceModuleTableView.as_view(), + name="device_module_sync", + ), + path( + "devices//install-module/", + InstallModuleView.as_view(), + name="install_module", + ), + path( + "devices//install-branch/", + InstallBranchView.as_view(), + name="install_branch", + ), path( "devices//ipaddress-sync/", DeviceIPAddressTableView.as_view(), @@ -335,5 +385,173 @@ InterfaceTypeMappingBulkDeleteView.as_view(), name="interfacetypemapping_bulk_delete", ), + # Device type mapping URLs + path( + "device-type-mappings/", + DeviceTypeMappingListView.as_view(), + name="devicetypemapping_list", + ), + path( + "device-type-mappings//", + DeviceTypeMappingView.as_view(), + name="devicetypemapping_detail", + ), + path( + "device-type-mappings/add/", + DeviceTypeMappingCreateView.as_view(), + name="devicetypemapping_add", + ), + path( + "device-type-mappings/import/", + DeviceTypeMappingBulkImportView.as_view(), + name="devicetypemapping_bulk_import", + ), + path( + "device-type-mappings//delete/", + DeviceTypeMappingDeleteView.as_view(), + name="devicetypemapping_delete", + ), + path( + "device-type-mappings//edit/", + DeviceTypeMappingEditView.as_view(), + name="devicetypemapping_edit", + ), + path( + "device-type-mappings//changelog/", + DeviceTypeMappingChangeLogView.as_view(), + name="devicetypemapping_changelog", + kwargs={"model": DeviceTypeMapping}, + ), + path( + "device-type-mappings/delete/", + DeviceTypeMappingBulkDeleteView.as_view(), + name="devicetypemapping_bulk_delete", + ), + # Module type mapping URLs + path( + "module-type-mappings/", + ModuleTypeMappingListView.as_view(), + name="moduletypemapping_list", + ), + path( + "module-type-mappings//", + ModuleTypeMappingView.as_view(), + name="moduletypemapping_detail", + ), + path( + "module-type-mappings/add/", + ModuleTypeMappingCreateView.as_view(), + name="moduletypemapping_add", + ), + path( + "module-type-mappings/import/", + ModuleTypeMappingBulkImportView.as_view(), + name="moduletypemapping_bulk_import", + ), + path( + "module-type-mappings//delete/", + ModuleTypeMappingDeleteView.as_view(), + name="moduletypemapping_delete", + ), + path( + "module-type-mappings//edit/", + ModuleTypeMappingEditView.as_view(), + name="moduletypemapping_edit", + ), + path( + "module-type-mappings//changelog/", + ModuleTypeMappingChangeLogView.as_view(), + name="moduletypemapping_changelog", + kwargs={"model": ModuleTypeMapping}, + ), + path( + "module-type-mappings/delete/", + ModuleTypeMappingBulkDeleteView.as_view(), + name="moduletypemapping_bulk_delete", + ), + # Module Bay Mapping URLs + path( + "module-bay-mappings/", + ModuleBayMappingListView.as_view(), + name="modulebaymapping_list", + ), + path( + "module-bay-mappings//", + ModuleBayMappingView.as_view(), + name="modulebaymapping_detail", + ), + path( + "module-bay-mappings/add/", + ModuleBayMappingCreateView.as_view(), + name="modulebaymapping_add", + ), + path( + "module-bay-mappings/import/", + ModuleBayMappingBulkImportView.as_view(), + name="modulebaymapping_bulk_import", + ), + path( + "module-bay-mappings//delete/", + ModuleBayMappingDeleteView.as_view(), + name="modulebaymapping_delete", + ), + path( + "module-bay-mappings//edit/", + ModuleBayMappingEditView.as_view(), + name="modulebaymapping_edit", + ), + path( + "module-bay-mappings//changelog/", + ModuleBayMappingChangeLogView.as_view(), + name="modulebaymapping_changelog", + kwargs={"model": ModuleBayMapping}, + ), + path( + "module-bay-mappings/delete/", + ModuleBayMappingBulkDeleteView.as_view(), + name="modulebaymapping_bulk_delete", + ), + # Normalization Rule URLs + path( + "normalization-rules/", + NormalizationRuleListView.as_view(), + name="normalizationrule_list", + ), + path( + "normalization-rules//", + NormalizationRuleView.as_view(), + name="normalizationrule_detail", + ), + path( + "normalization-rules/add/", + NormalizationRuleCreateView.as_view(), + name="normalizationrule_add", + ), + path( + "normalization-rules/import/", + NormalizationRuleBulkImportView.as_view(), + name="normalizationrule_bulk_import", + ), + path( + "normalization-rules//delete/", + NormalizationRuleDeleteView.as_view(), + name="normalizationrule_delete", + ), + path( + "normalization-rules//edit/", + NormalizationRuleEditView.as_view(), + name="normalizationrule_edit", + ), + path( + "normalization-rules//changelog/", + NormalizationRuleChangeLogView.as_view(), + name="normalizationrule_changelog", + kwargs={"model": NormalizationRule}, + ), + path( + "normalization-rules/delete/", + NormalizationRuleBulkDeleteView.as_view(), + name="normalizationrule_bulk_delete", + ), path("api/", include("netbox_librenms_plugin.api.urls")), ] diff --git a/netbox_librenms_plugin/utils.py b/netbox_librenms_plugin/utils.py index fee1d74d4e..0d1e8fcd03 100644 --- a/netbox_librenms_plugin/utils.py +++ b/netbox_librenms_plugin/utils.py @@ -1,18 +1,15 @@ +import logging import re from typing import Optional from dcim.models import Device +from django.db.models import Q from django.core.exceptions import ObjectDoesNotExist from django.http import HttpRequest from netbox.config import get_config from netbox.plugins import get_plugin_config from utilities.paginator import get_paginate_count as netbox_get_paginate_count - -import logging - -from django.db.models import Q - logger = logging.getLogger(__name__) @@ -273,7 +270,8 @@ def match_librenms_hardware_to_device_type(hardware_name: str) -> dict: """ Match LibreNMS hardware string to a NetBox DeviceType. - Only performs exact matching on part_number and model fields (case-insensitive). + Checks DeviceTypeMapping table first, then falls back to exact matching + on part_number and model fields (case-insensitive). Args: hardware_name (str): Hardware string from LibreNMS API (e.g., 'C9200L-48P-4X') @@ -282,13 +280,29 @@ def match_librenms_hardware_to_device_type(hardware_name: str) -> dict: dict: Dictionary containing: - matched (bool): Whether a match was found - device_type (DeviceType|None): The matched DeviceType object - - match_type (str|None): Always 'exact' if found, None otherwise + - match_type (str|None): 'mapping' if via DeviceTypeMapping, 'exact' if via + part_number/model, None otherwise """ from dcim.models import DeviceType + from netbox_librenms_plugin.models import DeviceTypeMapping + if not hardware_name or hardware_name == "-": return {"matched": False, "device_type": None, "match_type": None} + # Check DeviceTypeMapping table first + try: + mapping = DeviceTypeMapping.objects.get(librenms_hardware__iexact=hardware_name) + return { + "matched": True, + "device_type": mapping.netbox_device_type, + "match_type": "mapping", + } + except DeviceTypeMapping.DoesNotExist: + pass + except DeviceTypeMapping.MultipleObjectsReturned: + pass + # Try part number exact match try: device_type = DeviceType.objects.get(part_number__iexact=hardware_name) @@ -527,3 +541,109 @@ def check_vlan_group_matches( netbox_gid = netbox_tagged_group_ids.get(vid) return netbox_gid == selected_group_id return True + + +# Minimum NetBox version that supports {module_path} token in module templates + + +def supports_module_path(): + """Check if the running NetBox supports the {module_path} template token. + + Detects by checking for MODULE_PATH_TOKEN in dcim.constants rather than + comparing version strings — works with patched/pre-release builds too. + """ + try: + from dcim.constants import MODULE_PATH_TOKEN # noqa: F401 + + return True + except ImportError: + return False + + +def module_type_uses_module_path(module_type): + """Check if a ModuleType has any interface templates using {module_path}.""" + return any("{module_path}" in t.name for t in module_type.interfacetemplates.all()) + + +def has_nested_name_conflict(module_type, module_bay): + """Check if installing this module type in a nested bay would cause a name conflict. + + Returns True when ALL of the following are true: + - The module type has interface templates using only ``{module}`` (not ``{module_path}``) + - The bay is nested (its parent is owned by an installed module) + - There is at least one sibling bay under the same parent + + In this situation NetBox's ``resolve_name()`` replaces ``{module}`` with the + root ancestor's bay position, producing the same interface name for every + sibling at this nesting level. + """ + from dcim.constants import MODULE_TOKEN + + if not module_bay or not module_bay.module_id: + return False # Top-level bay — no conflict + + templates = list(module_type.interfacetemplates.all()) + if not templates: + return False # No interface templates + + uses_module_token = any(MODULE_TOKEN in t.name for t in templates) + if not uses_module_token: + return False # Template doesn't use {module} + + # Count how many unique interface names this template would produce across siblings + # If all siblings resolve to the same name, there's a conflict + from dcim.models import ModuleBay as ModuleBayModel + + sibling_count = ModuleBayModel.objects.filter( + device=module_bay.device, + module_id=module_bay.module_id, + ).count() + + return sibling_count > 1 + + +def apply_normalization_rules(value: str, scope: str, manufacturer=None) -> str: + """Apply NormalizationRule chain to transform a string before matching. + + Rules for the given scope are applied in priority order. Each rule's + regex substitution transforms the output of the previous rule, forming + a pipeline. If no rules match, the original value is returned unchanged. + + When *manufacturer* is given, manufacturer-scoped rules run first, + followed by unscoped (manufacturer=NULL) rules. When *manufacturer* + is ``None``, all rules for the scope run in priority order. + + Args: + value: The raw string to normalize (e.g. '3HE16474AARA01'). + scope: One of NormalizationRule.SCOPE_* constants. + manufacturer: Optional Manufacturer instance to scope rules. + + Returns: + The normalized string after all matching rules have been applied. + """ + from netbox_librenms_plugin.models import NormalizationRule + + if not value: + return value + + if manufacturer: + # Manufacturer-specific rules first, then unscoped rules + for mfg_filter in [{"manufacturer": manufacturer}, {"manufacturer__isnull": True}]: + rules = NormalizationRule.objects.filter(scope=scope, **mfg_filter).order_by("priority", "pk") + for rule in rules: + try: + value = re.sub(rule.match_pattern, rule.replacement, value) + except re.error: + logger.error( + "Invalid regex in NormalizationRule pk=%s pattern=%r — skipping", rule.pk, rule.match_pattern + ) + else: + rules = NormalizationRule.objects.filter(scope=scope).order_by("priority", "pk") + for rule in rules: + try: + value = re.sub(rule.match_pattern, rule.replacement, value) + except re.error: + logger.error( + "Invalid regex in NormalizationRule pk=%s pattern=%r — skipping", rule.pk, rule.match_pattern + ) + return value diff --git a/netbox_librenms_plugin/views/__init__.py b/netbox_librenms_plugin/views/__init__.py index df3beac79b..d8d04f16b1 100644 --- a/netbox_librenms_plugin/views/__init__.py +++ b/netbox_librenms_plugin/views/__init__.py @@ -10,6 +10,7 @@ from .base.interfaces_view import BaseInterfaceTableView # noqa: F401 from .base.ip_addresses_view import BaseIPAddressTableView, SingleIPAddressVerifyView # noqa: F401 from .base.librenms_sync_view import BaseLibreNMSSyncView # noqa: F401 +from .base.modules_view import InstallBranchView, InstallModuleView # noqa: F401 from .base.vlan_table_view import BaseVLANTableView # noqa: F401 from .imports import ( # noqa: F401 BulkImportConfirmView, @@ -24,6 +25,14 @@ SaveUserPrefView, ) from .mapping_views import ( # noqa: F401 + DeviceTypeMappingBulkDeleteView, + DeviceTypeMappingBulkImportView, + DeviceTypeMappingChangeLogView, + DeviceTypeMappingCreateView, + DeviceTypeMappingDeleteView, + DeviceTypeMappingEditView, + DeviceTypeMappingListView, + DeviceTypeMappingView, InterfaceTypeMappingBulkDeleteView, InterfaceTypeMappingBulkImportView, InterfaceTypeMappingChangeLogView, @@ -32,12 +41,37 @@ InterfaceTypeMappingEditView, InterfaceTypeMappingListView, InterfaceTypeMappingView, + ModuleBayMappingBulkDeleteView, + ModuleBayMappingBulkImportView, + ModuleBayMappingChangeLogView, + ModuleBayMappingCreateView, + ModuleBayMappingDeleteView, + ModuleBayMappingEditView, + ModuleBayMappingListView, + ModuleBayMappingView, + ModuleTypeMappingBulkDeleteView, + ModuleTypeMappingBulkImportView, + ModuleTypeMappingChangeLogView, + ModuleTypeMappingCreateView, + ModuleTypeMappingDeleteView, + ModuleTypeMappingEditView, + ModuleTypeMappingListView, + ModuleTypeMappingView, + NormalizationRuleBulkDeleteView, + NormalizationRuleBulkImportView, + NormalizationRuleChangeLogView, + NormalizationRuleCreateView, + NormalizationRuleDeleteView, + NormalizationRuleEditView, + NormalizationRuleListView, + NormalizationRuleView, ) from .object_sync import ( # noqa: F401 DeviceCableTableView, DeviceInterfaceTableView, DeviceIPAddressTableView, DeviceLibreNMSSyncView, + DeviceModuleTableView, DeviceVLANTableView, SaveVlanGroupOverridesView, SingleInterfaceVerifyView, diff --git a/netbox_librenms_plugin/views/base/cables_view.py b/netbox_librenms_plugin/views/base/cables_view.py index 4cbcdb7a36..46fd0f21b2 100644 --- a/netbox_librenms_plugin/views/base/cables_view.py +++ b/netbox_librenms_plugin/views/base/cables_view.py @@ -25,7 +25,6 @@ class BaseCableTableView(LibreNMSPermissionMixin, LibreNMSAPIMixin, CacheMixin, model = None # To be defined in subclasses partial_template_name = "netbox_librenms_plugin/_cable_sync_content.html" - interface_name_field = get_interface_name_field() def get_object(self, pk): """Retrieve the object (Device or VirtualMachine).""" @@ -54,11 +53,17 @@ def get_links_data(self, obj): if not success or "error" in data: return None + interface_name_field = get_interface_name_field(getattr(self, "request", None)) ports_data = self.get_ports_data(obj) local_ports_map = {} for port in ports_data.get("ports", []): - port_id = str(port["port_id"]) - port_name = port[self.interface_name_field] + raw_port_id = port.get("port_id") + if raw_port_id is None: + continue + port_id = str(raw_port_id) + port_name = port.get(interface_name_field) + if port_name is None: + continue local_ports_map[port_id] = port_name links = data.get("links", []) diff --git a/netbox_librenms_plugin/views/base/librenms_sync_view.py b/netbox_librenms_plugin/views/base/librenms_sync_view.py index 1346c3cb13..4a85cb1642 100644 --- a/netbox_librenms_plugin/views/base/librenms_sync_view.py +++ b/netbox_librenms_plugin/views/base/librenms_sync_view.py @@ -86,6 +86,7 @@ def get_context_data(self, request, obj): cable_context = self.get_cable_context(request, obj) ip_context = self.get_ip_context(request, obj) vlan_context = self.get_vlan_context(request, obj) + module_context = self.get_module_context(request, obj) interface_name_field = get_interface_name_field(request) @@ -103,6 +104,7 @@ def get_context_data(self, request, obj): "cable_sync": cable_context, "ip_sync": ip_context, "vlan_sync": vlan_context, + "module_sync": module_context, "v1v2form": AddToLIbreSNMPV1V2(prefix="v1v2"), "v3form": AddToLIbreSNMPV3(prefix="v3"), "librenms_device_id": self.librenms_id, @@ -230,6 +232,8 @@ def get_librenms_device_info(self, obj): if netbox_identities & librenms_identities: mismatched_device = False else: + # Device is still found (we have librenms_id), just mismatched + found_in_librenms = True mismatched_device = True librenms_device_details["netbox_dns_name"] = netbox_dns_name or "-" @@ -268,6 +272,13 @@ def get_vlan_context(self, request, obj): """ return None + def get_module_context(self, request, obj): + """ + Get the context data for module sync. + Subclasses should override this method if applicable. + """ + return None + @staticmethod def _strip_vc_pattern(name): """Strip the VC member naming suffix from a device name. diff --git a/netbox_librenms_plugin/views/base/modules_view.py b/netbox_librenms_plugin/views/base/modules_view.py new file mode 100644 index 0000000000..0a7317a51c --- /dev/null +++ b/netbox_librenms_plugin/views/base/modules_view.py @@ -0,0 +1,1064 @@ +from django.contrib import messages +from django.core.cache import cache +from django.db import transaction +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse +from django.utils import timezone +from django.views import View + +from netbox_librenms_plugin.views.mixins import ( + CacheMixin, + LibreNMSAPIMixin, + LibreNMSPermissionMixin, + NetBoxObjectPermissionMixin, +) + + +# entPhysicalClass values relevant for module sync +# Includes vendor-specific classes (Nokia TIMETRA-CHASSIS-MIB uses ioModule, cpmModule, etc.) +INVENTORY_CLASSES = { + "module", + "powerSupply", + "fan", + "port", + "container", + "ioModule", + "cpmModule", + "mdaModule", + "fabricModule", + "xioModule", +} + +# Model name values that indicate a generic/empty container (not real hardware) +_GENERIC_CONTAINER_MODELS = {"", "BUILTIN", "Default", "N/A"} + + +class BaseModuleTableView(LibreNMSPermissionMixin, LibreNMSAPIMixin, CacheMixin, View): + """ + Base view for synchronizing module/inventory data from LibreNMS. + Fetches inventory, matches against NetBox module bays and module types, + and renders a comparison table. + """ + + model = None + partial_template_name = "netbox_librenms_plugin/_module_sync_content.html" + + def get_object(self, pk): + """Retrieve the object (Device).""" + return get_object_or_404(self.model, pk=pk) + + def get_table(self, data, obj): + """Returns the table class. Subclasses should override.""" + raise NotImplementedError("Subclasses must implement get_table()") + + def post(self, request, pk): + """Fetch inventory from LibreNMS, cache it, and render the module sync table.""" + obj = self.get_object(pk) + + self.librenms_id = self.librenms_api.get_librenms_id(obj) + if not self.librenms_id: + messages.error(request, "Device not found in LibreNMS.") + return render( + request, + self.partial_template_name, + {"module_sync": {"object": obj, "table": None, "cache_expiry": None}}, + ) + + success, inventory_data = self.librenms_api.get_device_inventory(self.librenms_id) + + if not success: + messages.error(request, f"Failed to fetch inventory from LibreNMS: {inventory_data}") + return render( + request, + self.partial_template_name, + {"module_sync": {"object": obj, "table": None, "cache_expiry": None}}, + ) + + # Fetch transceiver data and merge with inventory + inventory_data = self._merge_transceiver_data(inventory_data) + + # Cache the merged inventory data + cache.set( + self.get_cache_key(obj, "inventory"), + inventory_data, + timeout=self.librenms_api.cache_timeout, + ) + + context = self._build_context(request, obj, inventory_data) + messages.success(request, "Inventory data refreshed successfully.") + return render(request, self.partial_template_name, {"module_sync": context}) + + def get_context_data(self, request, obj): + """Get context from cache (used by the main sync view on initial page load).""" + cached_data = cache.get(self.get_cache_key(obj, "inventory")) + if not cached_data: + return {"table": None, "object": obj, "cache_expiry": None} + return self._build_context(request, obj, cached_data) + + def _build_context(self, request, obj, inventory_data): + """Build context with matched inventory items and table.""" + # Build a lookup of all inventory items by index for parent resolution + index_map = {item["entPhysicalIndex"]: item for item in inventory_data} + + # Store manufacturer for normalization rules in _build_row + self._device_manufacturer = getattr(getattr(obj, "device_type", None), "manufacturer", None) + + # Get NetBox module bays and modules for this device + device_bays, module_scoped_bays = self._get_module_bays(obj) + module_types = self._get_module_types() + + # Collect top-level items and their sub-components + # Include synthetic transceiver items (from vendors without ENTITY-MIB SFP data) + # Exclude items that have any ancestor with an INVENTORY_CLASSES class + # (they appear as sub-components under that ancestor) + top_items = [] + for item in inventory_data: + if item.get("_from_transceiver_api"): + top_items.append(item) + continue + phys_class = item.get("entPhysicalClass") + if phys_class not in INVENTORY_CLASSES: + continue + # Skip items with generic model names (not real hardware). + # Containers with empty model are physical slot representations. + model = (item.get("entPhysicalModelName") or "").strip() + if phys_class == "container" and model in _GENERIC_CONTAINER_MODELS: + continue + if model and model in _GENERIC_CONTAINER_MODELS: + continue + # Walk up ancestor chain; skip if any ancestor is an inventory-class item. + # Containers with empty model are physical slot/bay representations, not + # real modules — skip them so children can be top-level items. + is_descendant = False + current_idx = item.get("entPhysicalContainedIn", 0) + for _ in range(10): + if not current_idx or current_idx not in index_map: + break + ancestor = index_map[current_idx] + anc_class = ancestor.get("entPhysicalClass") + if anc_class in INVENTORY_CLASSES: + anc_model = (ancestor.get("entPhysicalModelName") or "").strip() + # Empty-model containers are just physical slot representations + if anc_class == "container" and not anc_model: + current_idx = ancestor.get("entPhysicalContainedIn", 0) + continue + is_descendant = True + break + current_idx = ancestor.get("entPhysicalContainedIn", 0) + if is_descendant: + continue + top_items.append(item) + + table_data = [] + from netbox_librenms_plugin.utils import apply_normalization_rules + + # Build combined bay lookup so top-level items (including synthetic + # transceiver entries) can match bays created by installed modules. + all_bays = dict(device_bays) + for scope_bays in module_scoped_bays.values(): + all_bays.update(scope_bays) + + for item in top_items: + row = self._build_row(item, index_map, all_bays, module_types, depth=0) + parent_idx = len(table_data) + table_data.append(row) + + # Determine which bays sub-components should match against: + # If parent matched a bay with an installed module, use that module's child bays. + # If parent matched a bay but it's NOT installed, children can't be installed + # individually (parent must be installed first to create child bays). + parent_module_id = None + parent_bay_matched_but_uninstalled = False + if row.get("module_bay_id"): + matched_bay = all_bays.get(row["module_bay"]) + if matched_bay and hasattr(matched_bay, "installed_module") and matched_bay.installed_module: + parent_module_id = matched_bay.installed_module.pk + else: + # Parent matched a bay but it's not installed yet + parent_bay_matched_but_uninstalled = True + + if parent_bay_matched_but_uninstalled: + # Empty dict: children can't match any bay individually + child_bays = {} + elif parent_module_id: + child_bays = module_scoped_bays.get(parent_module_id, {}) + else: + child_bays = device_bays + + # Find sub-components with a model name (transceivers, converters, etc.) + # Track bay scope per depth level so nested modules use correct bays + bays_by_depth = {0: child_bays} + sub_items = self._get_sub_components(item["entPhysicalIndex"], inventory_data) + for depth, sub_item in sub_items: + scope_bays = bays_by_depth.get(depth, child_bays) + sub_row = self._build_row(sub_item, index_map, scope_bays, module_types, depth=depth) + table_data.append(sub_row) + + # If this sub-item matched an installed module, deeper items use its bays + if sub_row.get("module_bay_id"): + matched_sub_bay = scope_bays.get(sub_row["module_bay"]) + if ( + matched_sub_bay + and hasattr(matched_sub_bay, "installed_module") + and matched_sub_bay.installed_module + ): + sub_module_id = matched_sub_bay.installed_module.pk + bays_by_depth[depth + 1] = module_scoped_bays.get(sub_module_id, {}) + + # Mark parent if any child is installable + if sub_row.get("can_install"): + table_data[parent_idx]["has_installable_children"] = True + + # When parent is installable but children can't match bays yet + # (parent module not installed), enable "Install Branch" if children + # have matching module types (branch install handles bay creation) + if ( + parent_bay_matched_but_uninstalled + and row.get("can_install") + and not table_data[parent_idx].get("has_installable_children") + ): + for _depth, sub_item in sub_items: + sub_model = (sub_item.get("entPhysicalModelName") or "").strip() + if sub_model and ( + sub_model in module_types + or apply_normalization_rules( + sub_model, + "module_type", + manufacturer=getattr(self, "_device_manufacturer", None), + ) + in module_types + ): + table_data[parent_idx]["has_installable_children"] = True + break + + # Sort top-level groups by status, keeping children after their parent + table_data = self._sort_with_hierarchy(table_data) + + table = self.get_table(table_data, obj) + table.configure(request) + + cache_ttl = getattr(cache, "ttl", lambda k: None)(self.get_cache_key(obj, "inventory")) + cache_expiry = timezone.now() + timezone.timedelta(seconds=cache_ttl) if cache_ttl is not None else None + + return { + "table": table, + "object": obj, + "cache_expiry": cache_expiry, + } + + def _merge_transceiver_data(self, inventory_data): + """Merge transceiver API data with entity inventory. + + For vendors like Nokia that don't expose SFPs in ENTITY-MIB, + the transceiver API provides SFP model, serial, and type info. + + Strategy: + - For transceivers matching existing inventory items by entity_physical_index: + supplement entPhysicalModelName if empty + - For transceivers NOT in inventory: create synthetic inventory items + so they appear in the modules table + """ + success, transceivers = self.librenms_api.get_device_transceivers(self.librenms_id) + if not success or not transceivers: + return inventory_data + + # Build lookup of existing inventory items by index and serial + inv_by_index = {item["entPhysicalIndex"]: item for item in inventory_data} + inv_serials = { + (item.get("entPhysicalSerialNum") or "").strip() + for item in inventory_data + if (item.get("entPhysicalSerialNum") or "").strip() + } + + # Build port_id → ifName lookup for better synthetic item naming + port_name_map = self._build_port_name_map(transceivers) + + # Types that are containers, not real transceiver modules + SKIP_TYPES = {"Port Container", "Port", ""} + + for txr in transceivers: + ent_idx = txr.get("entity_physical_index") + if not ent_idx: + continue + + model = (txr.get("model") or "").strip() + serial = (txr.get("serial") or "").strip() + txr_type = (txr.get("type") or "").strip() + + # Skip containers and entries with no useful data + if txr_type in SKIP_TYPES and not model and not serial: + continue + + # Use transceiver type as model fallback (e.g., "CFP2/QSFP28") + display_model = model or (txr_type if txr_type not in SKIP_TYPES else "") + + if ent_idx in inv_by_index: + # Supplement existing inventory item if model is missing + existing = inv_by_index[ent_idx] + if not (existing.get("entPhysicalModelName") or "").strip() and display_model: + existing["entPhysicalModelName"] = display_model + if not (existing.get("entPhysicalSerialNum") or "").strip() and serial: + existing["entPhysicalSerialNum"] = serial + else: + # Skip if serial already exists in ENTITY-MIB data (avoid duplicates) + if serial and serial in inv_serials: + continue + # Create synthetic inventory item for SFPs not in entity inventory + port_id = txr.get("port_id", 0) + ifname = port_name_map.get(port_id) + if ifname: + name = ifname + elif port_id: + name = f"Transceiver (port {port_id})" + else: + name = f"Transceiver {ent_idx}" + + synthetic = { + "entPhysicalIndex": ent_idx, + "entPhysicalName": name, + "entPhysicalClass": "port", + "entPhysicalModelName": display_model, + "entPhysicalSerialNum": serial, + "entPhysicalDescr": txr_type, + "entPhysicalContainedIn": 0, + "_from_transceiver_api": True, + } + inventory_data.append(synthetic) + + return inventory_data + + def _build_port_name_map(self, transceivers): + """Build port_id → ifName mapping for transceiver ports. + + Fetches port data from LibreNMS to resolve port IDs to interface names, + enabling better bay matching for synthetic transceiver items (e.g., + Nokia 1/1/c1 instead of opaque port IDs). + """ + port_ids = {txr.get("port_id") for txr in transceivers if txr.get("port_id")} + if not port_ids: + return {} + + success, ports_data = self.librenms_api.get_ports(self.librenms_id) + if not success or not isinstance(ports_data, dict): + return {} + + return { + p["port_id"]: p["ifName"] + for p in ports_data.get("ports", []) + if p.get("port_id") in port_ids and p.get("ifName") + } + + def _get_sub_components(self, parent_idx, inventory_data): + """Find descendant items with a model name (real hardware, not empty containers). + + Returns list of (depth, item) tuples. + """ + results = [] + self._collect_descendants(parent_idx, inventory_data, depth=1, results=results) + return results + + def _collect_descendants(self, parent_idx, inventory_data, depth, results): + """Recursively collect descendant items that have a model name.""" + children = [i for i in inventory_data if i.get("entPhysicalContainedIn") == parent_idx] + for child in children: + model = (child.get("entPhysicalModelName") or "").strip() + if model and model not in _GENERIC_CONTAINER_MODELS: + results.append((depth, child)) + # Continue looking for deeper components (e.g., SFPs inside converters) + self._collect_descendants(child["entPhysicalIndex"], inventory_data, depth + 1, results) + else: + # Skip generic/empty items, but check their children + self._collect_descendants(child["entPhysicalIndex"], inventory_data, depth, results) + + def _sort_with_hierarchy(self, table_data): + """Sort table keeping children grouped under their parent.""" + status_order = {"Installed": 0, "Serial Mismatch": 1, "Matched": 2, "No Type": 3, "No Bay": 4, "Unmatched": 5} + + # Group into top-level items with their children + groups = [] + current_group = None + for row in table_data: + if row.get("depth", 0) == 0: + current_group = {"parent": row, "children": []} + groups.append(current_group) + elif current_group is not None: + current_group["children"].append(row) + + # Sort groups by parent status + groups.sort(key=lambda g: status_order.get(g["parent"]["status"], 99)) + + # Flatten back + result = [] + for group in groups: + result.append(group["parent"]) + result.extend(group["children"]) + return result + + def _get_module_bays(self, obj): + """Get module bays for the device, organized by scope. + + Returns: + tuple: (device_bays, module_bays) where: + - device_bays: {name: bay} for device-level bays (module=None) + - module_bays: {module_id: {name: bay}} for bays created by installed modules + """ + from dcim.models import ModuleBay + + bays = ModuleBay.objects.filter(device=obj).select_related("installed_module__module_type") + device_bays = {} + module_scoped_bays = {} + for bay in bays: + if bay.module_id: + module_scoped_bays.setdefault(bay.module_id, {})[bay.name] = bay + else: + device_bays[bay.name] = bay + return device_bays, module_scoped_bays + + def _get_module_types(self): + """Get all module types, indexed by model (part_number), with ModuleTypeMapping checked first.""" + from dcim.models import ModuleType + + from netbox_librenms_plugin.models import ModuleTypeMapping + + # Build base lookup from NetBox module types + types = ModuleType.objects.all().select_related("manufacturer") + result = {} + for mt in types: + result[mt.model] = mt + if mt.part_number and mt.part_number != mt.model: + result[mt.part_number] = mt + + # Overlay with explicit ModuleTypeMapping entries (take priority) + for mapping in ModuleTypeMapping.objects.select_related("netbox_module_type__manufacturer"): + result[mapping.librenms_model] = mapping.netbox_module_type + + return result + + def _find_parent_container_name(self, item, index_map): + """Resolve the parent container name for an inventory item.""" + contained_in = item.get("entPhysicalContainedIn", 0) + if contained_in == 0: + return None + parent = index_map.get(contained_in) + if parent: + return parent.get("entPhysicalName", "") + return None + + def _match_module_bay(self, item, index_map, module_bays): + """ + Try to match an inventory item to a NetBox ModuleBay. + Checks ModuleBayMapping table first (exact then regex), then falls back + to exact parent name match, then positional matching. + """ + import re + + from netbox_librenms_plugin.models import ModuleBayMapping + + parent_name = self._find_parent_container_name(item, index_map) + item_name = item.get("entPhysicalName", "") + item_descr = item.get("entPhysicalDescr", "") + phys_class = item.get("entPhysicalClass", "") + + # Build candidate names: parent, item name, item description + candidate_names = [n for n in [parent_name, item_name, item_descr] if n] + + # Check ModuleBayMapping table for each candidate (exact match) + for name in candidate_names: + filters = {"librenms_name": name, "is_regex": False} + if phys_class: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class=phys_class).first() + if not mapping: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + else: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + + if mapping and mapping.netbox_bay_name in module_bays: + return module_bays[mapping.netbox_bay_name] + + # Regex pattern matching on all candidate names + for name in candidate_names: + bay = self._lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping) + if bay and self._fpc_slot_matches(name, bay): + return bay + + # Fallback: exact match on candidate names against bay dict + for name in candidate_names: + if name in module_bays: + return module_bays[name] + + # Positional fallback: determine slot number from container sibling order + # Handles SFPs inside converters where containers are unnamed + bay = self._match_bay_by_position(item, index_map, module_bays) + if bay: + return bay + + return None + + @staticmethod + def _fpc_slot_matches(candidate_name, bay): + """Validate that a regex-matched bay's parent slot position is consistent with + a positional descriptor like 'Model @ FPC/pic/port'. + + Returns True if the descriptor has no FPC reference, or if the bay's parent + module slot position matches the FPC number in the descriptor. Prevents + orphaned top-level items (e.g. QSFP @ 1/1/1 when FPC1 is not installed) + from incorrectly matching bays belonging to a different FPC's module. + """ + import re as _re + + match = _re.search(r"@\s+(\d+)/", candidate_name) + if not match: + return True + expected_fpc = match.group(1) + module = getattr(bay, "module", None) + if not module: + return True + parent_bay = getattr(module, "module_bay", None) + if not parent_bay: + return True + return parent_bay.position == expected_fpc + + @staticmethod + def _lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping): + """Try regex ModuleBayMapping patterns against a name. + + Returns matched module bay or None. + """ + regex_filters = {"is_regex": True} + if phys_class: + regex_mappings = list(ModuleBayMapping.objects.filter(**regex_filters, librenms_class=phys_class)) + list( + ModuleBayMapping.objects.filter(**regex_filters, librenms_class="") + ) + else: + regex_mappings = list(ModuleBayMapping.objects.filter(**regex_filters, librenms_class="")) + + for mapping in regex_mappings: + try: + match = re.fullmatch(mapping.librenms_name, name) + except re.error: + continue + if match: + resolved_bay = match.expand(mapping.netbox_bay_name) + if resolved_bay in module_bays: + bay = module_bays[resolved_bay] + if BaseModuleTableView._fpc_slot_matches(name, bay): + return bay + return None + + @staticmethod + def _match_bay_by_position(item, index_map, module_bays): + """Match bay by item's positional order among container siblings. + + When an item is inside a container (no model), walk up to find the + nearest ancestor with a model, count which container slot the item + occupies, and match to the bay by number (e.g., SFP 1, SFP 2). + """ + # Walk up through modelless containers to find the parent with a model + current_idx = item.get("entPhysicalContainedIn", 0) + container_idx = None + for _ in range(5): + if not current_idx or current_idx not in index_map: + return None + ancestor = index_map[current_idx] + model = (ancestor.get("entPhysicalModelName") or "").strip() + if model: + # Found the parent with a model; container_idx is the intermediate container + break + container_idx = current_idx + current_idx = ancestor.get("entPhysicalContainedIn", 0) + else: + return None + + if not container_idx: + return None + + # Determine position: count siblings of the container under the parent + parent_with_model_idx = current_idx + siblings = sorted( + [i for i in index_map.values() if i.get("entPhysicalContainedIn") == parent_with_model_idx], + key=lambda x: x.get("entPhysicalParentRelPos", 0), + ) + slot_num = None + for i, sib in enumerate(siblings): + if sib["entPhysicalIndex"] == container_idx: + slot_num = i + 1 + break + + if slot_num is None: + return None + + # Try common bay naming patterns + for pattern in [f"SFP {slot_num}", f"Slot {slot_num}", f"Bay {slot_num}", f"Port {slot_num}"]: + if pattern in module_bays: + return module_bays[pattern] + + return None + + def _build_row(self, item, index_map, module_bays, module_types, depth=0): + """Build a single table row from a LibreNMS inventory item.""" + from netbox_librenms_plugin.utils import ( + apply_normalization_rules, + has_nested_name_conflict, + module_type_uses_module_path, + supports_module_path, + ) + + model_name = item.get("entPhysicalModelName", "") or "" + serial = item.get("entPhysicalSerialNum", "") or "" + phys_class = item.get("entPhysicalClass", "") + name = item.get("entPhysicalName", "") or "-" + description = item.get("entPhysicalDescr", "") or "" + + # Match to NetBox module bay + matched_bay = self._match_module_bay(item, index_map, module_bays) + + # Match to NetBox module type (direct lookup, then normalization fallback) + matched_type = module_types.get(model_name) if model_name else None + if not matched_type and model_name: + normalized = apply_normalization_rules( + model_name, "module_type", manufacturer=getattr(self, "_device_manufacturer", None) + ) + if normalized != model_name: + matched_type = module_types.get(normalized) + + # Check {module_path} compatibility + needs_module_path = matched_type and module_type_uses_module_path(matched_type) + module_path_blocked = needs_module_path and not supports_module_path() + + # Check for nested module naming conflicts + name_conflict = ( + matched_type + and matched_bay + and not module_path_blocked + and has_nested_name_conflict(matched_type, matched_bay) + ) + + # Determine status + status = self._determine_status(matched_bay, matched_type, serial, module_path_blocked) + + row = { + "name": name, + "model": model_name or "-", + "serial": serial or "-", + "description": description, + "item_class": phys_class, + "module_bay": matched_bay.name if matched_bay else "-", + "module_type": matched_type.model if matched_type else "-", + "status": status, + "row_class": "", + "can_install": False, + "module_bay_id": matched_bay.pk if matched_bay else None, + "module_type_id": matched_type.pk if matched_type else None, + "depth": depth, + "ent_physical_index": item.get("entPhysicalIndex"), + "has_installable_children": False, + } + + if module_path_blocked: + row["row_class"] = "table-warning" + row["module_path_warning"] = ( + "This module type uses {module_path} in its interface template " + "but the running NetBox does not support it yet." + ) + + if name_conflict: + row["row_class"] = "table-warning" + row["name_conflict_warning"] = ( + "This module type uses {module} in its interface template. " + "Installing multiple siblings will create duplicate interface names. " + "An interface naming plugin with a rewrite rule for this module type can resolve this." + ) + + # Add URLs for matched objects + if matched_bay: + row["module_bay_url"] = matched_bay.get_absolute_url() + # Check if a module is already installed in this bay + if hasattr(matched_bay, "installed_module") and matched_bay.installed_module: + installed = matched_bay.installed_module + row["installed_module"] = installed + row["module_url"] = installed.get_absolute_url() + # Check serial match + if serial and installed.serial and installed.serial.strip() == serial.strip(): + status = "Installed" + row["row_class"] = "table-success" + elif serial and installed.serial and installed.serial.strip() != serial.strip(): + status = "Serial Mismatch" + row["row_class"] = "table-danger" + else: + status = "Installed" + row["row_class"] = "table-success" + row["status"] = status + elif matched_type and not module_path_blocked: + # Bay exists, type matched, no module installed → can install + row["can_install"] = True + + if matched_type: + row["module_type_url"] = matched_type.get_absolute_url() + + return row + + def _determine_status(self, matched_bay, matched_type, serial, module_path_blocked=False): + """Determine the sync status for an inventory item.""" + if module_path_blocked: + return "Requires Upgrade" + if matched_bay and matched_type: + return "Matched" + if not matched_bay: + return "No Bay" + if not matched_type: + return "No Type" + return "Unmatched" + + +class InstallModuleView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, View): + """Install a NetBox Module into a ModuleBay from LibreNMS inventory data.""" + + def post(self, request, pk): + from dcim.models import Device, Module, ModuleBay, ModuleType + + self.required_object_permissions = {"POST": [("add", Module)]} + if error := self.require_all_permissions_json("POST"): + return error + + device = get_object_or_404(Device, pk=pk) + module_bay_id = request.POST.get("module_bay_id") + module_type_id = request.POST.get("module_type_id") + serial = request.POST.get("serial", "").strip() + + if not module_bay_id or not module_type_id: + messages.error(request, "Missing module bay or module type.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + module_bay = get_object_or_404(ModuleBay, pk=module_bay_id, device=device) + module_type = get_object_or_404(ModuleType, pk=module_type_id) + + # Block install if module type uses {module_path} and NetBox doesn't support it + from netbox_librenms_plugin.utils import module_type_uses_module_path, supports_module_path + + if module_type_uses_module_path(module_type) and not supports_module_path(): + messages.error( + request, + f"Cannot install {module_type.model}: its interface templates use " + f"{{module_path}} which this NetBox version does not support.", + ) + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Check if bay already has a module installed + if hasattr(module_bay, "installed_module") and module_bay.installed_module: + messages.warning(request, f"Module bay '{module_bay.name}' already has a module installed.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + try: + with transaction.atomic(): + module = Module( + device=device, + module_bay=module_bay, + module_type=module_type, + serial=serial, + status="active", + ) + module.full_clean() + module.save() + + messages.success( + request, f"Installed {module_type.model} in {module_bay.name} (serial: {serial or 'N/A'})." + ) + except Exception as e: + messages.error(request, f"Failed to install module: {e}") + + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + +class InstallBranchView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, CacheMixin, View): + """Install a module and all its installable descendants from LibreNMS inventory.""" + + def post(self, request, pk): + from dcim.models import Device, Module, ModuleBay, ModuleType + + self.required_object_permissions = {"POST": [("add", Module)]} + if error := self.require_all_permissions_json("POST"): + return error + + device = get_object_or_404(Device, pk=pk) + parent_index = request.POST.get("parent_index") + + if not parent_index: + messages.error(request, "Missing parent inventory index.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + try: + parent_index = int(parent_index) + except ValueError: + messages.error(request, "Invalid parent inventory index.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Get cached inventory data + cached_data = cache.get(self.get_cache_key(device, "inventory")) + if not cached_data: + messages.error(request, "No cached inventory data. Please refresh modules first.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Build index map and collect the branch to install + index_map = {item["entPhysicalIndex"]: item for item in cached_data} + branch_items = self._collect_branch(parent_index, cached_data) + + if not branch_items: + messages.warning(request, "No installable items found in this branch.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Load module types (with mappings) + module_types = self._get_module_types() + + # Install top-down: each install may create new child bays + installed = [] + skipped = [] + failed = [] + + try: + with transaction.atomic(): + for item in branch_items: + result = self._install_single( + device, + item, + index_map, + module_types, + ModuleBay, + ModuleType, + Module, + ) + if result["status"] == "installed": + installed.append(result["name"]) + elif result["status"] == "skipped": + skipped.append(f"{result['name']}: {result['reason']}") + else: + failed.append(f"{result['name']}: {result['reason']}") + except Exception as e: + messages.error(request, f"Branch install failed: {e}") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Report results + if installed: + messages.success(request, f"Installed {len(installed)} module(s): {', '.join(installed)}") + if skipped: + messages.info(request, f"Skipped {len(skipped)}: {'; '.join(skipped)}") + if failed: + messages.warning(request, f"Failed {len(failed)}: {'; '.join(failed)}") + + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + def _collect_branch(self, parent_index, inventory_data): + """Collect all items in a branch depth-first, parent first. + + Returns items in install order (parent before children). + """ + items = [] + parent = next((i for i in inventory_data if i["entPhysicalIndex"] == parent_index), None) + if parent: + model = (parent.get("entPhysicalModelName") or "").strip() + if model: + items.append(parent) + self._collect_children(parent_index, inventory_data, items) + return items + + def _collect_children(self, parent_idx, inventory_data, items): + """Recursively collect children with models, depth-first.""" + children = [i for i in inventory_data if i.get("entPhysicalContainedIn") == parent_idx] + for child in children: + model = (child.get("entPhysicalModelName") or "").strip() + if model: + items.append(child) + # Always recurse to find deeper items (containers may lack models) + self._collect_children(child["entPhysicalIndex"], inventory_data, items) + + def _get_module_types(self): + """Get all module types indexed by model, with mappings applied.""" + from dcim.models import ModuleType + + from netbox_librenms_plugin.models import ModuleTypeMapping + + types = ModuleType.objects.all().select_related("manufacturer") + result = {} + for mt in types: + result[mt.model] = mt + if mt.part_number and mt.part_number != mt.model: + result[mt.part_number] = mt + for mapping in ModuleTypeMapping.objects.select_related("netbox_module_type__manufacturer"): + result[mapping.librenms_model] = mapping.netbox_module_type + return result + + def _install_single(self, device, item, index_map, module_types, ModuleBay, ModuleType, Module): + """Try to install a single inventory item. + + Re-fetches module bays each time since parent installs create new ones. + Scopes bay lookup to the correct parent module to handle duplicate bay names. + """ + from netbox_librenms_plugin.models import ModuleBayMapping + from netbox_librenms_plugin.utils import ( + apply_normalization_rules, + module_type_uses_module_path, + supports_module_path, + ) + + model_name = (item.get("entPhysicalModelName") or "").strip() + serial = (item.get("entPhysicalSerialNum") or "").strip() + name = item.get("entPhysicalName", "") or model_name + + # Match module type (direct, then normalization fallback) + matched_type = module_types.get(model_name) + if not matched_type and model_name: + manufacturer = getattr(getattr(device, "device_type", None), "manufacturer", None) + normalized = apply_normalization_rules(model_name, "module_type", manufacturer=manufacturer) + if normalized != model_name: + matched_type = module_types.get(normalized) + if not matched_type: + return {"status": "skipped", "name": name, "reason": "no matching type"} + + # Check {module_path} compatibility + if module_type_uses_module_path(matched_type) and not supports_module_path(): + return {"status": "skipped", "name": name, "reason": "requires {module_path}"} + + # Re-fetch module bays (parent install creates new child bays) + bays = ModuleBay.objects.filter(device=device).select_related("installed_module__module_type") + + # Determine if this item belongs under an installed module + # by tracing its LibreNMS parent hierarchy to an installed item + parent_module_id = self._find_parent_module_id(item, index_map, device, ModuleBay) + + if parent_module_id: + bay_dict = {bay.name: bay for bay in bays if bay.module_id == parent_module_id} + else: + bay_dict = {bay.name: bay for bay in bays if not bay.module_id} + + # Match module bay using mapping table + matched_bay = self._match_bay(item, index_map, bay_dict, ModuleBayMapping) + if not matched_bay: + return {"status": "skipped", "name": name, "reason": "no matching bay"} + + # Check if already installed + if hasattr(matched_bay, "installed_module") and matched_bay.installed_module: + return {"status": "skipped", "name": name, "reason": "bay already occupied"} + + # Install + try: + with transaction.atomic(): # savepoint: failure here won't abort parent tx + module = Module( + device=device, + module_bay=matched_bay, + module_type=matched_type, + serial=serial, + status="active", + ) + module.full_clean() + module.save() + except Exception as e: + error_msg = str(e) + if "dcim_interface_unique_device_name" in error_msg: + error_msg = ( + "duplicate interface name — this module type's interface template " + "uses {module} which resolves to the same name for all siblings. " + "An interface naming plugin with a rewrite rule for this module type can fix this." + ) + return {"status": "failed", "name": name, "reason": error_msg} + + return {"status": "installed", "name": f"{matched_type.model} → {matched_bay.name}"} + + @staticmethod + def _find_parent_module_id(item, index_map, device, ModuleBay): + """Find the NetBox module ID for the installed parent of this inventory item. + + Walks up the LibreNMS hierarchy to find an ancestor whose name matches + an installed module bay on the device. + """ + from netbox_librenms_plugin.models import ModuleBayMapping + + current = item + for _ in range(10): # max depth guard + parent_idx = current.get("entPhysicalContainedIn", 0) + if not parent_idx or parent_idx not in index_map: + return None + parent = index_map[parent_idx] + parent_name = parent.get("entPhysicalName", "") + parent_descr = parent.get("entPhysicalDescr", "") + + # Check if this parent matches an installed module bay on the device + device_bays = ModuleBay.objects.filter(device=device, module_id__isnull=True).select_related( + "installed_module" + ) + + for bay in device_bays: + if hasattr(bay, "installed_module") and bay.installed_module: + if bay.name == parent_name or (parent_descr and bay.name == parent_descr): + return bay.installed_module.pk + + # Also check ModuleBayMapping for indirect matches + for name in [parent_name, parent_descr]: + if not name: + continue + mapping = ModuleBayMapping.objects.filter(librenms_name=name).first() + if mapping: + bay = ( + ModuleBay.objects.filter(device=device, name=mapping.netbox_bay_name, module_id__isnull=True) + .select_related("installed_module") + .first() + ) + if bay and hasattr(bay, "installed_module") and bay.installed_module: + return bay.installed_module.pk + + current = parent + return None + + @staticmethod + def _match_bay(item, index_map, module_bays, ModuleBayMapping): + """Match an inventory item to a module bay (same logic as BaseModuleTableView).""" + import re + + # Resolve parent name + contained_in = item.get("entPhysicalContainedIn", 0) + parent_name = None + if contained_in: + parent = index_map.get(contained_in) + if parent: + parent_name = parent.get("entPhysicalName", "") + + item_name = item.get("entPhysicalName", "") + item_descr = item.get("entPhysicalDescr", "") + phys_class = item.get("entPhysicalClass", "") + + # Build candidate names: parent, item name, item description + candidate_names = [n for n in [parent_name, item_name, item_descr] if n] + + # Check mapping for each candidate (exact match) + for name in candidate_names: + filters = {"librenms_name": name, "is_regex": False} + if phys_class: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class=phys_class).first() + if not mapping: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + else: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + if mapping and mapping.netbox_bay_name in module_bays: + return module_bays[mapping.netbox_bay_name] + + # Regex pattern matching on all candidate names + for name in candidate_names: + bay = BaseModuleTableView._lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping) + if bay: + return bay + + # Fallback: exact match on candidate names against bay dict + for name in candidate_names: + if name in module_bays: + return module_bays[name] + + # Positional fallback for items inside converters + return BaseModuleTableView._match_bay_by_position(item, index_map, module_bays) diff --git a/netbox_librenms_plugin/views/mapping_views.py b/netbox_librenms_plugin/views/mapping_views.py index b1fcec9c77..55ff7bd658 100644 --- a/netbox_librenms_plugin/views/mapping_views.py +++ b/netbox_librenms_plugin/views/mapping_views.py @@ -1,14 +1,44 @@ from netbox.views import generic from utilities.views import register_model_view -from netbox_librenms_plugin.filters import InterfaceTypeMappingFilterSet +from netbox_librenms_plugin.filters import ( + DeviceTypeMappingFilterSet, + InterfaceTypeMappingFilterSet, + ModuleBayMappingFilterSet, + ModuleTypeMappingFilterSet, + NormalizationRuleFilterSet, +) from netbox_librenms_plugin.forms import ( + DeviceTypeMappingFilterForm, + DeviceTypeMappingForm, + DeviceTypeMappingImportForm, InterfaceTypeMappingFilterForm, InterfaceTypeMappingForm, InterfaceTypeMappingImportForm, + ModuleBayMappingFilterForm, + ModuleBayMappingForm, + ModuleBayMappingImportForm, + ModuleTypeMappingFilterForm, + ModuleTypeMappingForm, + ModuleTypeMappingImportForm, + NormalizationRuleFilterForm, + NormalizationRuleForm, + NormalizationRuleImportForm, +) +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) +from netbox_librenms_plugin.tables.mappings import ( + DeviceTypeMappingTable, + InterfaceTypeMappingTable, + ModuleBayMappingTable, + ModuleTypeMappingTable, + NormalizationRuleTable, ) -from netbox_librenms_plugin.models import InterfaceTypeMapping -from netbox_librenms_plugin.tables.mappings import InterfaceTypeMappingTable from netbox_librenms_plugin.views.mixins import LibreNMSPermissionMixin @@ -84,3 +114,243 @@ class InterfaceTypeMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectC """ queryset = InterfaceTypeMapping.objects.all() + + +# --- DeviceTypeMapping views --- + + +class DeviceTypeMappingListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all DeviceTypeMapping objects.""" + + queryset = DeviceTypeMapping.objects.all() + table = DeviceTypeMappingTable + filterset = DeviceTypeMappingFilterSet + filterset_form = DeviceTypeMappingFilterForm + template_name = "netbox_librenms_plugin/devicetypemapping_list.html" + + +class DeviceTypeMappingCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + form = DeviceTypeMappingForm + + +@register_model_view(DeviceTypeMapping, "bulk_import", path="import", detail=False) +class DeviceTypeMappingBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing DeviceTypeMapping objects.""" + + queryset = DeviceTypeMapping.objects.all() + model_form = DeviceTypeMappingImportForm + + +class DeviceTypeMappingView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + + +class DeviceTypeMappingEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + form = DeviceTypeMappingForm + + +class DeviceTypeMappingDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + + +class DeviceTypeMappingBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple DeviceTypeMapping objects.""" + + queryset = DeviceTypeMapping.objects.all() + table = DeviceTypeMappingTable + + +class DeviceTypeMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + + +# --- ModuleTypeMapping views --- + + +class ModuleTypeMappingListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all ModuleTypeMapping objects.""" + + queryset = ModuleTypeMapping.objects.all() + table = ModuleTypeMappingTable + filterset = ModuleTypeMappingFilterSet + filterset_form = ModuleTypeMappingFilterForm + template_name = "netbox_librenms_plugin/moduletypemapping_list.html" + + +class ModuleTypeMappingCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + form = ModuleTypeMappingForm + + +@register_model_view(ModuleTypeMapping, "bulk_import", path="import", detail=False) +class ModuleTypeMappingBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing ModuleTypeMapping objects.""" + + queryset = ModuleTypeMapping.objects.all() + model_form = ModuleTypeMappingImportForm + + +class ModuleTypeMappingView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + + +class ModuleTypeMappingEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + form = ModuleTypeMappingForm + + +class ModuleTypeMappingDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + + +class ModuleTypeMappingBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple ModuleTypeMapping objects.""" + + queryset = ModuleTypeMapping.objects.all() + table = ModuleTypeMappingTable + + +class ModuleTypeMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + + +# --- ModuleBayMapping views --- + + +class ModuleBayMappingListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all ModuleBayMapping objects.""" + + queryset = ModuleBayMapping.objects.all() + table = ModuleBayMappingTable + filterset = ModuleBayMappingFilterSet + filterset_form = ModuleBayMappingFilterForm + template_name = "netbox_librenms_plugin/modulebaymapping_list.html" + + +class ModuleBayMappingCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + form = ModuleBayMappingForm + + +@register_model_view(ModuleBayMapping, "bulk_import", path="import", detail=False) +class ModuleBayMappingBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing ModuleBayMapping objects.""" + + queryset = ModuleBayMapping.objects.all() + model_form = ModuleBayMappingImportForm + + +class ModuleBayMappingView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + + +class ModuleBayMappingEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + form = ModuleBayMappingForm + + +class ModuleBayMappingDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + + +class ModuleBayMappingBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple ModuleBayMapping objects.""" + + queryset = ModuleBayMapping.objects.all() + table = ModuleBayMappingTable + + +class ModuleBayMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + + +# --- NormalizationRule views --- + + +class NormalizationRuleListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all NormalizationRule objects.""" + + queryset = NormalizationRule.objects.all() + table = NormalizationRuleTable + filterset = NormalizationRuleFilterSet + filterset_form = NormalizationRuleFilterForm + template_name = "netbox_librenms_plugin/normalizationrule_list.html" + + +class NormalizationRuleCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + form = NormalizationRuleForm + + +@register_model_view(NormalizationRule, "bulk_import", path="import", detail=False) +class NormalizationRuleBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing NormalizationRule objects.""" + + queryset = NormalizationRule.objects.all() + model_form = NormalizationRuleImportForm + + +class NormalizationRuleView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + + +class NormalizationRuleEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + form = NormalizationRuleForm + + +class NormalizationRuleDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + + +class NormalizationRuleBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple NormalizationRule objects.""" + + queryset = NormalizationRule.objects.all() + table = NormalizationRuleTable + + +class NormalizationRuleChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() diff --git a/netbox_librenms_plugin/views/object_sync/__init__.py b/netbox_librenms_plugin/views/object_sync/__init__.py index e9893cf7d8..f025cb2a7b 100644 --- a/netbox_librenms_plugin/views/object_sync/__init__.py +++ b/netbox_librenms_plugin/views/object_sync/__init__.py @@ -5,6 +5,7 @@ DeviceInterfaceTableView, DeviceIPAddressTableView, DeviceLibreNMSSyncView, + DeviceModuleTableView, DeviceVLANTableView, SaveVlanGroupOverridesView, SingleInterfaceVerifyView, diff --git a/netbox_librenms_plugin/views/object_sync/devices.py b/netbox_librenms_plugin/views/object_sync/devices.py index 429a3119a8..fe12c42751 100644 --- a/netbox_librenms_plugin/views/object_sync/devices.py +++ b/netbox_librenms_plugin/views/object_sync/devices.py @@ -17,6 +17,7 @@ LibreNMSInterfaceTable, VCInterfaceTable, ) +from netbox_librenms_plugin.tables.modules import LibreNMSModuleTable from netbox_librenms_plugin.utils import ( get_interface_name_field, get_missing_vlan_warning, @@ -29,6 +30,7 @@ from ..base.interfaces_view import BaseInterfaceTableView from ..base.ip_addresses_view import BaseIPAddressTableView from ..base.librenms_sync_view import BaseLibreNMSSyncView +from ..base.modules_view import BaseModuleTableView from ..base.vlan_table_view import BaseVLANTableView from ..mixins import CacheMixin, LibreNMSPermissionMixin @@ -63,6 +65,12 @@ def get_vlan_context(self, request, obj): vlan_table_view.request = request return vlan_table_view.get_vlan_context(request, obj) + def get_module_context(self, request, obj): + """Return module sync context for the device.""" + module_table_view = DeviceModuleTableView() + module_table_view.request = request + return module_table_view.get_context_data(request, obj) + class DeviceInterfaceTableView(BaseInterfaceTableView): """Interface synchronization table for Devices.""" @@ -384,3 +392,15 @@ class DeviceVLANTableView(BaseVLANTableView): """VLAN synchronization table view for Devices.""" model = Device + + +class DeviceModuleTableView(BaseModuleTableView): + """Module/inventory synchronization view for Devices.""" + + model = Device + + def get_table(self, data, obj): + """Return the module sync table.""" + table = LibreNMSModuleTable(data, device=obj) + table.htmx_url = f"{self.request.path}?tab=modules" + return table diff --git a/netbox_librenms_plugin/views/sync/cables.py b/netbox_librenms_plugin/views/sync/cables.py index 0e52f3b017..27dc5943c2 100644 --- a/netbox_librenms_plugin/views/sync/cables.py +++ b/netbox_librenms_plugin/views/sync/cables.py @@ -1,3 +1,5 @@ +import logging + from dcim.models import Cable, Device, Interface from django.contrib import messages from django.core.cache import cache @@ -9,6 +11,8 @@ from netbox_librenms_plugin.views.mixins import CacheMixin, LibreNMSPermissionMixin, NetBoxObjectPermissionMixin +logger = logging.getLogger(__name__) + class SyncCablesView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, CacheMixin, View): """Create NetBox cables using cached LibreNMS link data.""" @@ -42,7 +46,11 @@ def get_cached_links_data(self, request, obj): return cached_data.get("links", []) def create_cable(self, local_interface, remote_interface, request): - """Create a cable between local and remote interfaces.""" + """Create a cable between local and remote interfaces. + + Returns: + True on success, False on failure. + """ try: Cable.objects.create( a_terminations=[local_interface], @@ -81,13 +89,12 @@ def process_single_interface(self, interface, cached_links): link_data = next(link for link in cached_links if link["local_port"] == interface["interface"]) return self.handle_cable_creation(link_data, interface) except StopIteration: - return {"status": "invalid"} + return {"status": "invalid", "interface": interface.get("interface", "")} def verify_cable_creation_requirements(self, link_data): """Return True if all required NetBox IDs are present in link data.""" required_fields = [ "netbox_local_interface_id", - "netbox_remote_device_id", "netbox_remote_interface_id", ] @@ -113,13 +120,21 @@ def handle_cable_creation(self, link_data, interface): return {"status": "missing_remote", "interface": interface["interface"]} def process_interface_sync(self, selected_interfaces, cached_links): - """Process cable sync for all selected interfaces and return results.""" + """Process cable sync for all selected interfaces and return results. + + Each interface is processed in its own atomic block so individual + failures roll back only that cable without affecting others. + """ results = {"valid": [], "invalid": [], "duplicate": [], "missing_remote": []} - with transaction.atomic(): - for interface in selected_interfaces: - result = self.process_single_interface(interface, cached_links) + for interface in selected_interfaces: + try: + with transaction.atomic(): + result = self.process_single_interface(interface, cached_links) results[result["status"]].append(result.get("interface", "")) + except Exception: + logger.exception("Failed to sync cable for interface %s", interface.get("interface", "")) + results["invalid"].append(interface.get("interface", "")) return results diff --git a/netbox_librenms_plugin/views/sync/device_fields.py b/netbox_librenms_plugin/views/sync/device_fields.py index 29eed67c2a..d0a7141709 100644 --- a/netbox_librenms_plugin/views/sync/device_fields.py +++ b/netbox_librenms_plugin/views/sync/device_fields.py @@ -1,7 +1,7 @@ from dcim.models import Device, Manufacturer, Platform from django.contrib import messages from django.core.exceptions import ValidationError -from django.db import IntegrityError +from django.db import IntegrityError, transaction from django.shortcuts import get_object_or_404, redirect from django.views import View @@ -278,26 +278,28 @@ def post(self, request, pk): pass try: - platform = Platform.objects.create( - name=platform_name, - manufacturer=manufacturer, - ) - except IntegrityError: - messages.error( - request, - f"Platform '{platform_name}' could not be created (slug collision). Try a different name.", - ) + with transaction.atomic(): + platform = Platform.objects.create( + name=platform_name, + manufacturer=manufacturer, + ) + + device.platform = platform + device.full_clean() + device.save() + except IntegrityError as e: + error_str = str(e) + if "platform" in error_str.lower() or "slug" in error_str.lower(): + messages.error( + request, + f"Platform '{platform_name}' could not be created (slug collision). Try a different name.", + ) + else: + messages.error(request, f"Failed to assign platform '{platform_name}': {error_str}") return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) - - old_platform = device.platform - device.platform = platform - try: - device.full_clean() - device.save() - except (ValidationError, IntegrityError) as e: - device.platform = old_platform + except ValidationError as e: error_msg = e.message_dict if hasattr(e, "message_dict") else str(e) - messages.error(request, f"Failed to assign platform '{platform}': {error_msg}") + messages.error(request, f"Failed to assign platform '{platform_name}': {error_msg}") return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) messages.success( diff --git a/netbox_librenms_plugin/views/sync/devices.py b/netbox_librenms_plugin/views/sync/devices.py index da0f9af5b0..eb45b375ff 100644 --- a/netbox_librenms_plugin/views/sync/devices.py +++ b/netbox_librenms_plugin/views/sync/devices.py @@ -26,7 +26,7 @@ def get_object(self, object_id): try: return Device.objects.get(pk=object_id) except Device.DoesNotExist: - return VirtualMachine.objects.get(pk=object_id) + return get_object_or_404(VirtualMachine, pk=object_id) def post(self, request, object_id): """Add a device to LibreNMS using the submitted SNMP form.""" diff --git a/netbox_librenms_plugin/views/sync/interfaces.py b/netbox_librenms_plugin/views/sync/interfaces.py index 62f7e9eb3a..1bc5babf02 100644 --- a/netbox_librenms_plugin/views/sync/interfaces.py +++ b/netbox_librenms_plugin/views/sync/interfaces.py @@ -164,14 +164,8 @@ def sync_interface(self, obj, librenms_interface, exclude_columns, interface_nam ) # Sync VLANs if not excluded - vlan_synced = False if "vlans" not in exclude_columns: self._sync_interface_vlans(interface, librenms_interface, interface_name) - vlan_synced = True - - # Skip redundant save when _sync_interface_vlans already saved (via _update_interface_vlan_assignment) - if not vlan_synced: - interface.save() def get_netbox_interface_type(self, librenms_interface): """Return the NetBox interface type mapped from LibreNMS type and speed.""" diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 0000000000..62c68691ec --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,6 @@ +"""Conftest for e2e tests — no Django initialization needed.""" + +import os + +# Prevent pytest-django from trying to initialize Django +os.environ.pop("DJANGO_SETTINGS_MODULE", None) diff --git a/tests/e2e/test_module_install.py b/tests/e2e/test_module_install.py new file mode 100644 index 0000000000..cee05eb904 --- /dev/null +++ b/tests/e2e/test_module_install.py @@ -0,0 +1,330 @@ +"""End-to-end Playwright tests for LibreNMS plugin module sync workflow. + +These tests exercise the full import → modules → install flow against a +live NetBox + LibreNMS instance inside the devcontainer. + +Prerequisites: + - NetBox running at NETBOX_URL (default http://172.22.0.4:8000) + - LibreNMS server configured in plugin settings + - Device 15 (WS-C4900M) exists and is linked to LibreNMS + - Playwright installed: pip install playwright && playwright install chromium + +Run: + cd /home/mzieba/workspace/netbox-librenms-plugin + HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= \ + no_proxy=localhost,127.0.0.1,172.22.0.4 \ + python -m pytest tests/e2e/test_module_install.py -v -s +""" + +import os +import subprocess +import time + +import pytest + +NETBOX_URL = os.environ.get("NETBOX_URL", "http://172.22.0.4:8000") +NETBOX_USER = os.environ.get("NETBOX_USER", "admin") +NETBOX_PASS = os.environ.get("NETBOX_PASS", "admin") +CONTAINER_NAME = None + + +def _get_container(): + """Find the devcontainer name.""" + global CONTAINER_NAME + if CONTAINER_NAME: + return CONTAINER_NAME + result = subprocess.run( + ["docker", "ps", "--format", "{{.Names}}"], + capture_output=True, + text=True, + ) + for name in result.stdout.strip().split("\n"): + if "devcontainer-devcontainer" in name: + CONTAINER_NAME = name + return name + pytest.skip("No devcontainer found") + + +def _netbox_shell(code): + """Run Python code in NetBox's Django shell.""" + import shlex + + container = _get_container() + escaped = shlex.quote(code) + result = subprocess.run( + [ + "docker", + "exec", + container, + "bash", + "-c", + f"cd /opt/netbox/netbox && python3 manage.py shell -c {escaped}", + ], + capture_output=True, + text=True, + env={"PATH": "/usr/bin:/bin", "HOME": "/root"}, + ) + # Filter out config loading lines + lines = [line for line in result.stdout.strip().split("\n") if not line.startswith(("🧬", "156 objects"))] + return "\n".join(lines).strip() + + +def _delete_device_modules(device_id): + """Remove all modules from a device.""" + _netbox_shell( + f"from dcim.models import Module; " + f"deleted = Module.objects.filter(device_id={device_id}).delete(); " + f"print(f'Deleted {{deleted}}')" + ) + + +def _get_interfaces(device_id): + """Get interface names for a device.""" + output = _netbox_shell( + f"from dcim.models import Interface; " + f'[print(f\'{{i.name}}|{{i.module.module_type.model if i.module else "-"}}|' + f'{{i.module.module_bay.name if i.module else "-"}}\')' + f" for i in Interface.objects.filter(device_id={device_id}).order_by('name')]" + ) + results = [] + for line in output.split("\n"): + if "|" in line: + name, mod_type, bay = line.split("|") + results.append({"name": name, "module_type": mod_type, "bay": bay}) + return results + + +@pytest.fixture(scope="module") +def browser(): + """Launch browser for the test module.""" + from playwright.sync_api import sync_playwright + + pw = sync_playwright().start() + b = pw.chromium.launch(headless=True) + yield b + b.close() + pw.stop() + + +@pytest.fixture +def page(browser): + """Create a new page and log in to NetBox.""" + ctx = browser.new_context(ignore_https_errors=True) + pg = ctx.new_page() + + pg.goto(f"{NETBOX_URL}/login/", timeout=10000) + pg.fill("#id_username", NETBOX_USER) + pg.fill("#id_password", NETBOX_PASS) + pg.click("button[type=submit]") + pg.wait_for_load_state("networkidle") + yield pg + ctx.close() + + +class TestModuleInstallWorkflow: + """Test the full module sync and install workflow on device 15 (WS-C4900M).""" + + DEVICE_ID = 15 + + def _goto_modules_tab(self, page): + """Navigate to the modules sync tab and refresh data.""" + page.goto(f"{NETBOX_URL}/dcim/devices/{self.DEVICE_ID}/librenms-sync/?tab=modules") + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Click Refresh Modules + btn = page.query_selector('button:has-text("Refresh Modules")') + assert btn is not None, "Refresh Modules button not found" + btn.click() + time.sleep(8) + + def _get_table_rows(self, page): + """Parse the module sync table into dicts.""" + pane = page.query_selector("#modules") + assert pane is not None, "Modules pane not found" + + rows = [] + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if len(cells) >= 8: + rows.append( + { + "name": cells[0].inner_text().strip(), + "model": cells[1].inner_text().strip(), + "serial": cells[2].inner_text().strip(), + "bay": cells[5].inner_text().strip(), + "type": cells[6].inner_text().strip(), + "status": cells[7].inner_text().strip(), + } + ) + return rows + + def test_clean_state_shows_install_buttons(self, page): + """After deleting all modules, table shows Install buttons.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + rows = self._get_table_rows(page) + assert len(rows) > 0, "No rows in module sync table" + + # Top-level items with matched bays should show Matched status + supervisor = [r for r in rows if "Supervisor(slot 1)" in r["name"]] + assert len(supervisor) == 1, f"Expected 1 Supervisor row, got {len(supervisor)}" + assert supervisor[0]["status"] == "Matched", f"Expected Matched, got {supervisor[0]['status']}" + + def test_single_install(self, page): + """Installing a single top-level module works.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + # Install FanTray 1 + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "FanTray 1" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install"):not(:has-text("Branch"))') + if btn: + btn.click() + time.sleep(5) + break + + # Verify via DB + output = _netbox_shell( + f"from dcim.models import Module; " + f"m = Module.objects.filter(device_id={self.DEVICE_ID}, module_bay__name='Fan Tray 1').first(); " + f"print(m.module_type.model if m else 'NONE')" + ) + assert "WS-X4992" in output, f"FanTray not installed: {output}" + + def test_branch_install_supervisor(self, page): + """Branch install creates supervisor + X2 transceivers with correct names.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + # Click Install Branch on Supervisor(slot 1) + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Supervisor(slot 1)" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install Branch")') + assert btn is not None, "Install Branch button not found for Supervisor" + btn.click() + break + + # Wait for branch install to complete (creates many modules + signals) + time.sleep(20) + page.wait_for_load_state("networkidle") + time.sleep(5) + + # Verify interfaces have correct names (not bare position numbers) + interfaces = _get_interfaces(self.DEVICE_ID) + x2_interfaces = [i for i in interfaces if i["module_type"] in ("X2-10GB-LR", "X2-10GB-SR")] + + assert len(x2_interfaces) > 0, "No X2 transceiver interfaces created" + + for iface in x2_interfaces: + assert iface["name"].startswith("TenGigabitEthernet"), ( + f"Interface '{iface['name']}' in {iface['bay']} " + f"should start with 'TenGigabitEthernet' (INR rule not applied?)" + ) + + def test_branch_install_no_duplicate_errors(self, page): + """Branch install handles already-occupied bays gracefully.""" + # Don't delete modules — some should already be installed + self._goto_modules_tab(page) + + # Click Install Branch on Supervisor(slot 1) again + pane = page.query_selector("#modules") + branch_btn = None + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Supervisor(slot 1)" in cells[0].inner_text(): + branch_btn = tr.query_selector('button:has-text("Install Branch")') + break + + if branch_btn: + branch_btn.click() + time.sleep(10) + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Check for error messages — should only have skips, no failures + body_text = page.query_selector("body").inner_text() + assert "Branch install failed" not in body_text, ( + "Branch install crashed instead of handling errors gracefully" + ) + + def test_child_bays_hidden_when_parent_not_installed(self, page): + """Children show 'No Bay' when parent module is not installed.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + rows = self._get_table_rows(page) + + # Children of Supervisor(slot 1) should show "No matching bay" + # since Supervisor isn't installed, its child bays don't exist yet + children = [r for r in rows if r["name"].startswith("└─") and "TenGigabitEthernet1/" in r["name"]] + for child in children: + assert "No matching bay" in child["bay"], ( + f"Child '{child['name']}' should show 'No matching bay' when parent not installed, got '{child['bay']}'" + ) + + def test_full_workflow(self, page): + """Full workflow: clean → install individuals → branch install → verify.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + # Step 1: Install PSUs and FanTray individually + for label in ["FanTray 1", "Power Supply 1", "Power Supply 2"]: + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and label in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install"):not(:has-text("Branch"))') + if btn: + btn.click() + time.sleep(4) + break + + # Step 2: Branch install Supervisor + transceivers + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Supervisor(slot 1)" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install Branch")') + if btn: + btn.click() + time.sleep(10) + break + + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Step 3: Branch install Linecard + self._goto_modules_tab(page) + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Linecard(slot 3)" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install Branch")') + if btn: + btn.click() + time.sleep(10) + break + + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Verify: all installable modules should be installed + self._goto_modules_tab(page) + rows = self._get_table_rows(page) + + matched_but_not_installed = [r for r in rows if r["status"] == "Matched" and not r["name"].startswith("└─")] + assert len(matched_but_not_installed) == 0, ( + f"Top-level items still 'Matched' after full workflow: {[r['name'] for r in matched_but_not_installed]}" + ) + + # Verify interface naming + interfaces = _get_interfaces(self.DEVICE_ID) + for iface in interfaces: + assert iface["name"] != "1", "Interface with bare name '1' found — INR rule not applied" From 38611672d2144a21441d194a515a6957cffc4f10 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 1 Mar 2026 17:28:33 +0100 Subject: [PATCH 12/39] fix: migrate librenms_id CF type to json; fix Install Branch logic - __init__.py: create librenms_id custom field as type 'json' (was 'integer'); auto-migrate existing integer-typed fields to 'json' on post_migrate so the multi-server dict format is accepted by the UI - modules_view.py: pass device_bays (not all_bays) to _build_row for ENTITY-MIB top-level items to prevent module-scoped bay name collisions with device-level bays of the same name - modules_view.py: suppress Install Branch when all candidate children have module types that require {module_path} on an unsupported NetBox version; now checks module_type_uses_module_path + supports_module_path before setting has_installable_children --- netbox_librenms_plugin/__init__.py | 21 +++++++---- .../views/base/modules_view.py | 35 ++++++++++++------- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/netbox_librenms_plugin/__init__.py b/netbox_librenms_plugin/__init__.py index d0499c53f3..dc933681a1 100644 --- a/netbox_librenms_plugin/__init__.py +++ b/netbox_librenms_plugin/__init__.py @@ -70,15 +70,15 @@ def _validate_legacy_config(self, plugin_config): def _ensure_librenms_id_custom_field(sender, **kwargs): """ - Auto-create the 'librenms_id' custom field if it doesn't exist. + Auto-create (or migrate) the 'librenms_id' custom field. Runs after migrations via post_migrate signal to ensure tables exist. Uses dispatch_uid to avoid duplicate connections. + + librenms_id stores a per-server JSON mapping {"server_key": device_id}. + Legacy installations may have this field typed as 'integer'; we upgrade it + to 'json' automatically so the UI and API accept the dict format. """ # Only run once per migrate invocation (post_migrate fires per-app). - # The _executed flag is intentionally never reset: migrations are expected to - # run in short-lived CLI processes (manage.py migrate) where the flag is - # naturally cleared on exit. Long-running processes (e.g. gunicorn workers) - # should not rely on this handler re-executing after startup. if getattr(_ensure_librenms_id_custom_field, "_executed", False): return _ensure_librenms_id_custom_field._executed = True # not reset; see comment above @@ -93,7 +93,7 @@ def _ensure_librenms_id_custom_field(sender, **kwargs): cf, created = CustomField.objects.get_or_create( name="librenms_id", defaults={ - "type": "integer", + "type": "json", "label": "LibreNMS ID", "description": "LibreNMS Device ID for synchronization (auto-created by plugin)", "required": False, @@ -103,6 +103,15 @@ def _ensure_librenms_id_custom_field(sender, **kwargs): }, ) + # Migrate legacy integer-typed field to JSON so the multi-server + # dict format {"server_key": device_id} is accepted by the UI/API. + if not created and cf.type == "integer": + cf.type = "json" + cf.save(update_fields=["type"]) + logging.getLogger("netbox_librenms_plugin").info( + "Migrated 'librenms_id' custom field type from integer to json" + ) + # Ensure the field is assigned to the required object types from dcim.models import Device, Interface from virtualization.models import VirtualMachine, VMInterface diff --git a/netbox_librenms_plugin/views/base/modules_view.py b/netbox_librenms_plugin/views/base/modules_view.py index 0a7317a51c..3434498392 100644 --- a/netbox_librenms_plugin/views/base/modules_view.py +++ b/netbox_librenms_plugin/views/base/modules_view.py @@ -150,16 +150,25 @@ def _build_context(self, request, obj, inventory_data): top_items.append(item) table_data = [] - from netbox_librenms_plugin.utils import apply_normalization_rules + from netbox_librenms_plugin.utils import ( + apply_normalization_rules, + module_type_uses_module_path, + supports_module_path, + ) - # Build combined bay lookup so top-level items (including synthetic - # transceiver entries) can match bays created by installed modules. + # Build combined bay lookup so synthetic transceiver entries (which may + # live inside installed modules) can find their module-scoped bays. all_bays = dict(device_bays) for scope_bays in module_scoped_bays.values(): all_bays.update(scope_bays) for item in top_items: - row = self._build_row(item, index_map, all_bays, module_types, depth=0) + # Transceiver API entries may live inside installed modules, so they + # need the full bay map. ENTITY-MIB top-level items must only match + # device-level bays to avoid name collisions with module-scoped bays + # that share the same name as a device bay. + item_bays = all_bays if item.get("_from_transceiver_api") else device_bays + row = self._build_row(item, index_map, item_bays, module_types, depth=0) parent_idx = len(table_data) table_data.append(row) @@ -170,7 +179,7 @@ def _build_context(self, request, obj, inventory_data): parent_module_id = None parent_bay_matched_but_uninstalled = False if row.get("module_bay_id"): - matched_bay = all_bays.get(row["module_bay"]) + matched_bay = item_bays.get(row["module_bay"]) if matched_bay and hasattr(matched_bay, "installed_module") and matched_bay.installed_module: parent_module_id = matched_bay.installed_module.pk else: @@ -210,8 +219,8 @@ def _build_context(self, request, obj, inventory_data): table_data[parent_idx]["has_installable_children"] = True # When parent is installable but children can't match bays yet - # (parent module not installed), enable "Install Branch" if children - # have matching module types (branch install handles bay creation) + # (parent module not installed), enable "Install Branch" only if + # children have matching module types that are not module_path_blocked. if ( parent_bay_matched_but_uninstalled and row.get("can_install") @@ -219,15 +228,17 @@ def _build_context(self, request, obj, inventory_data): ): for _depth, sub_item in sub_items: sub_model = (sub_item.get("entPhysicalModelName") or "").strip() - if sub_model and ( - sub_model in module_types - or apply_normalization_rules( + if not sub_model: + continue + matched = module_types.get(sub_model) + if not matched: + normalized = apply_normalization_rules( sub_model, "module_type", manufacturer=getattr(self, "_device_manufacturer", None), ) - in module_types - ): + matched = module_types.get(normalized) + if matched and not (module_type_uses_module_path(matched) and not supports_module_path()): table_data[parent_idx]["has_installable_children"] = True break From 4ecce1037defd54ff185b1feb8478457d9bc9ce3 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 1 Mar 2026 17:52:24 +0100 Subject: [PATCH 13/39] refactor: module_path badges are informational only; never block install MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Never block module installation based on {module_path} compatibility. The old 'Requires Upgrade' status and can_install=False guard are replaced with informational badges: * module_path_warning → 'Upgrade NetBox' warning icon with tooltip (module uses {module_path} but current NetBox does not support it; installation proceeds, interface naming may not be perfect) * module_type_upgrade_hint → info icon on end modules still using {module} when the running NetBox already supports {module_path} (suggests updating the module type's interface templates) - Add module_type_uses_module_token() and module_type_is_end_module() helpers to utils.py. - Remove module_path_blocked guards from InstallModuleView.post() and InstallBranchView._install_single() — install always proceeds. - Revert Install Branch heuristic: any matched child module type enables the button regardless of {module_path} support. - _determine_status() no longer returns 'Requires Upgrade'. --- netbox_librenms_plugin/tables/modules.py | 17 +++- netbox_librenms_plugin/utils.py | 14 ++++ .../views/base/modules_view.py | 79 ++++++++----------- 3 files changed, 63 insertions(+), 47 deletions(-) diff --git a/netbox_librenms_plugin/tables/modules.py b/netbox_librenms_plugin/tables/modules.py index 458364fe10..5a4a6e7cb8 100644 --- a/netbox_librenms_plugin/tables/modules.py +++ b/netbox_librenms_plugin/tables/modules.py @@ -114,12 +114,18 @@ def render_status(self, value, record): "No Type": "bg-warning", "Unmatched": "bg-secondary", "Serial Mismatch": "bg-danger", - "Requires Upgrade": "bg-warning", "Name Conflict": "bg-warning", } badge_class = badge_classes.get(value, "bg-secondary") if warning := record.get("module_path_warning"): - return format_html('{}', badge_class, warning, value) + return format_html( + '{}' + ' ', + badge_class, + warning, + value, + "Upgrade NetBox to fully support {module_path}", + ) if warning := record.get("name_conflict_warning"): return format_html( '{}' @@ -129,6 +135,13 @@ def render_status(self, value, record): value, warning, ) + if hint := record.get("module_type_upgrade_hint"): + return format_html( + '{} ', + badge_class, + value, + hint, + ) return format_html('{}', badge_class, value) def render_actions(self, value, record): diff --git a/netbox_librenms_plugin/utils.py b/netbox_librenms_plugin/utils.py index 0d1e8fcd03..5a305494b3 100644 --- a/netbox_librenms_plugin/utils.py +++ b/netbox_librenms_plugin/utils.py @@ -565,6 +565,20 @@ def module_type_uses_module_path(module_type): return any("{module_path}" in t.name for t in module_type.interfacetemplates.all()) +def module_type_uses_module_token(module_type) -> bool: + """Check if a ModuleType has interface templates using the {module} token.""" + try: + from dcim.constants import MODULE_TOKEN + except ImportError: + return False + return any(MODULE_TOKEN in t.name for t in module_type.interfacetemplates.all()) + + +def module_type_is_end_module(module_type) -> bool: + """Return True if this module type defines no module bays (i.e., it is a leaf/end module).""" + return not module_type.modulebays.exists() + + def has_nested_name_conflict(module_type, module_bay): """Check if installing this module type in a nested bay would cause a name conflict. diff --git a/netbox_librenms_plugin/views/base/modules_view.py b/netbox_librenms_plugin/views/base/modules_view.py index 3434498392..68a94b2fca 100644 --- a/netbox_librenms_plugin/views/base/modules_view.py +++ b/netbox_librenms_plugin/views/base/modules_view.py @@ -150,11 +150,7 @@ def _build_context(self, request, obj, inventory_data): top_items.append(item) table_data = [] - from netbox_librenms_plugin.utils import ( - apply_normalization_rules, - module_type_uses_module_path, - supports_module_path, - ) + from netbox_librenms_plugin.utils import apply_normalization_rules # Build combined bay lookup so synthetic transceiver entries (which may # live inside installed modules) can find their module-scoped bays. @@ -219,8 +215,8 @@ def _build_context(self, request, obj, inventory_data): table_data[parent_idx]["has_installable_children"] = True # When parent is installable but children can't match bays yet - # (parent module not installed), enable "Install Branch" only if - # children have matching module types that are not module_path_blocked. + # (parent module not installed), enable "Install Branch" if any child + # has a matching module type (branch install handles bay creation). if ( parent_bay_matched_but_uninstalled and row.get("can_install") @@ -238,7 +234,7 @@ def _build_context(self, request, obj, inventory_data): manufacturer=getattr(self, "_device_manufacturer", None), ) matched = module_types.get(normalized) - if matched and not (module_type_uses_module_path(matched) and not supports_module_path()): + if matched: table_data[parent_idx]["has_installable_children"] = True break @@ -610,7 +606,9 @@ def _build_row(self, item, index_map, module_bays, module_types, depth=0): from netbox_librenms_plugin.utils import ( apply_normalization_rules, has_nested_name_conflict, + module_type_is_end_module, module_type_uses_module_path, + module_type_uses_module_token, supports_module_path, ) @@ -632,20 +630,23 @@ def _build_row(self, item, index_map, module_bays, module_types, depth=0): if normalized != model_name: matched_type = module_types.get(normalized) - # Check {module_path} compatibility + # Badge flags — purely informational, never block installation needs_module_path = matched_type and module_type_uses_module_path(matched_type) - module_path_blocked = needs_module_path and not supports_module_path() - - # Check for nested module naming conflicts - name_conflict = ( + # {module_path} used but NetBox version does not support it → "Upgrade NetBox" hint + netbox_upgrade_needed = bool(needs_module_path and not supports_module_path()) + # End module still using old {module} when {module_path} is available → "Upgrade module-type" hint + suggest_type_upgrade = bool( matched_type - and matched_bay - and not module_path_blocked - and has_nested_name_conflict(matched_type, matched_bay) + and supports_module_path() + and module_type_is_end_module(matched_type) + and module_type_uses_module_token(matched_type) ) + # Check for nested module naming conflicts + name_conflict = matched_type and matched_bay and has_nested_name_conflict(matched_type, matched_bay) + # Determine status - status = self._determine_status(matched_bay, matched_type, serial, module_path_blocked) + status = self._determine_status(matched_bay, matched_type, serial) row = { "name": name, @@ -665,11 +666,21 @@ def _build_row(self, item, index_map, module_bays, module_types, depth=0): "has_installable_children": False, } - if module_path_blocked: + if netbox_upgrade_needed: row["row_class"] = "table-warning" row["module_path_warning"] = ( - "This module type uses {module_path} in its interface template " - "but the running NetBox does not support it yet." + "This module type uses {module_path} in its interface templates. " + "The current NetBox version does not support {module_path} yet — " + "installation will proceed but interface naming may not work as expected. " + "Upgrade NetBox to enable full {module_path} support." + ) + + if suggest_type_upgrade: + row["module_type_upgrade_hint"] = ( + "This module type uses {module} in its interface templates. " + "Since this NetBox version supports {module_path}, consider updating " + "the module type's interface templates to use {module_path} for " + "precise per-slot interface naming." ) if name_conflict: @@ -699,7 +710,7 @@ def _build_row(self, item, index_map, module_bays, module_types, depth=0): status = "Installed" row["row_class"] = "table-success" row["status"] = status - elif matched_type and not module_path_blocked: + elif matched_type: # Bay exists, type matched, no module installed → can install row["can_install"] = True @@ -708,10 +719,8 @@ def _build_row(self, item, index_map, module_bays, module_types, depth=0): return row - def _determine_status(self, matched_bay, matched_type, serial, module_path_blocked=False): + def _determine_status(self, matched_bay, matched_type, serial): """Determine the sync status for an inventory item.""" - if module_path_blocked: - return "Requires Upgrade" if matched_bay and matched_type: return "Matched" if not matched_bay: @@ -744,18 +753,6 @@ def post(self, request, pk): module_bay = get_object_or_404(ModuleBay, pk=module_bay_id, device=device) module_type = get_object_or_404(ModuleType, pk=module_type_id) - # Block install if module type uses {module_path} and NetBox doesn't support it - from netbox_librenms_plugin.utils import module_type_uses_module_path, supports_module_path - - if module_type_uses_module_path(module_type) and not supports_module_path(): - messages.error( - request, - f"Cannot install {module_type.model}: its interface templates use " - f"{{module_path}} which this NetBox version does not support.", - ) - sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) - return redirect(f"{sync_url}?tab=modules#librenms-module-table") - # Check if bay already has a module installed if hasattr(module_bay, "installed_module") and module_bay.installed_module: messages.warning(request, f"Module bay '{module_bay.name}' already has a module installed.") @@ -914,11 +911,7 @@ def _install_single(self, device, item, index_map, module_types, ModuleBay, Modu Scopes bay lookup to the correct parent module to handle duplicate bay names. """ from netbox_librenms_plugin.models import ModuleBayMapping - from netbox_librenms_plugin.utils import ( - apply_normalization_rules, - module_type_uses_module_path, - supports_module_path, - ) + from netbox_librenms_plugin.utils import apply_normalization_rules model_name = (item.get("entPhysicalModelName") or "").strip() serial = (item.get("entPhysicalSerialNum") or "").strip() @@ -934,10 +927,6 @@ def _install_single(self, device, item, index_map, module_types, ModuleBay, Modu if not matched_type: return {"status": "skipped", "name": name, "reason": "no matching type"} - # Check {module_path} compatibility - if module_type_uses_module_path(matched_type) and not supports_module_path(): - return {"status": "skipped", "name": name, "reason": "requires {module_path}"} - # Re-fetch module bays (parent install creates new child bays) bays = ModuleBay.objects.filter(device=device).select_related("installed_module__module_type") From 7ac54bb04323a14d2ea9e43d976dd78993bf955b Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 1 Mar 2026 23:29:37 +0100 Subject: [PATCH 14/39] fix: use modulebaytemplates instead of modulebays in module_type_is_end_module --- netbox_librenms_plugin/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox_librenms_plugin/utils.py b/netbox_librenms_plugin/utils.py index 5a305494b3..2c76b5494f 100644 --- a/netbox_librenms_plugin/utils.py +++ b/netbox_librenms_plugin/utils.py @@ -575,8 +575,8 @@ def module_type_uses_module_token(module_type) -> bool: def module_type_is_end_module(module_type) -> bool: - """Return True if this module type defines no module bays (i.e., it is a leaf/end module).""" - return not module_type.modulebays.exists() + """Return True if this module type defines no module bay templates (i.e., it is a leaf/end module).""" + return not module_type.modulebaytemplates.exists() def has_nested_name_conflict(module_type, module_bay): From 7d0ec6c699596f2abd57434d094e73578ff84c32 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 1 Mar 2026 23:29:08 +0100 Subject: [PATCH 15/39] =?UTF-8?q?feat:=20legacy=20librenms=5Fid=20int?= =?UTF-8?q?=E2=86=92JSON=20migration=20action=20on=20import=20page?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add migrate_legacy_librenms_id(obj, server_key) helper in utils.py Converts a bare-integer librenms_id to {server_key: int_value}. Returns True if migration happened, False if already JSON or absent. Does not call save() — caller is responsible. - Detect legacy int format in validate_device_for_import() When find_by_librenms_id matches a Device or VM whose librenms_id CF is still a bare integer, set result['librenms_id_needs_migration']=True so the import page can surface a migration action. - Add 'migrate_librenms_id' action in DeviceConflictActionView Verifies CF is still an int, requires serial_confirmed or force checkbox, calls migrate_legacy_librenms_id() + save(). - Fix collision check to use find_by_librenms_id instead of raw queryset The old Device.objects.filter(custom_field_data__librenms_id=int(...)) only matched the legacy integer format; the new helper matches both the JSON dict format and the legacy format. - Show 'Legacy ID format' badge + 'Migrate ID format' button in template Appears in the existing_match_type=='librenms_id' section when librenms_id_needs_migration is set. Button is disabled (requires force checkbox) when serial not confirmed. - Add TestLegacyLibreNMSIdMigration test class (6 tests) - Update TestDeviceConflictActionView mocks for new filter().first() chain --- .../import_utils/device_operations.py | 9 ++ .../htmx/device_validation_details.html | 26 +++ .../tests/test_import_utils.py | 151 ++++++++++++++++++ netbox_librenms_plugin/utils.py | 33 ++++ .../views/imports/actions.py | 34 +++- 5 files changed, 247 insertions(+), 6 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index 8a7c37527d..161bd8e84b 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -198,6 +198,7 @@ def validate_device_for_import( "serial_action": None, # None, "link", "conflict", "update_serial", "hostname_differs" "serial_confirmed": False, # True when librenms_id match and serial matches "serial_duplicate": False, # True when incoming serial is already on a different device + "librenms_id_needs_migration": False, # True when librenms_id is still a legacy bare int "name_matches": False, # True when existing device name matches LibreNMS sysName "name_sync_available": False, # True when existing device name differs from sysName "suggested_name": None, # sysName to suggest when name_sync_available is True @@ -270,6 +271,10 @@ def validate_device_for_import( result["import_as_vm"] = True # Force VM mode since VM exists result["can_import"] = False + # Detect legacy bare-integer format so UI can offer a migration action + if isinstance(existing_vm.custom_field_data.get("librenms_id"), int): + result["librenms_id_needs_migration"] = True + # Check if name matches resolved name (accounts for use_sysname/strip_domain) # Note: name_sync_available/suggested_name are intentionally not set for VMs # because UpdateDeviceNameView only supports Device objects; VM name-sync @@ -293,6 +298,10 @@ def validate_device_for_import( result["existing_match_type"] = "librenms_id" result["can_import"] = False + # Detect legacy bare-integer format so UI can offer a migration action + if isinstance(existing_device.custom_field_data.get("librenms_id"), int): + result["librenms_id_needs_migration"] = True + # Check if name matches resolved name (accounts for use_sysname/strip_domain) if hostname and existing_device.name == hostname: result["name_matches"] = True diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html index f2f181da13..11ae76fcbd 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html @@ -376,7 +376,33 @@
Device Information
{% if validation.device_type_mismatch %} Type mismatch {% endif %} + {% if validation.librenms_id_needs_migration %} + Legacy ID format + {% endif %}
+ {% if validation.librenms_id_needs_migration %} +
+
+ {% csrf_token %} + + + {% if not validation.serial_confirmed %} +
+ + +
+ {% endif %} + +
+
+ {% endif %} {% elif validation.existing_match_type == 'hostname' %}
diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 0f0b7a5c39..1b8df20727 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1614,6 +1614,151 @@ def device_filter(**kwargs): assert result["device_type_mismatch"] is False +class TestLegacyLibreNMSIdMigration: + """Test detection of legacy bare-integer librenms_id format during device validation.""" + + PATCHES = [ + "netbox_librenms_plugin.import_utils.device_operations.Site", + "netbox_librenms_plugin.import_utils.device_operations.Rack", + "netbox_librenms_plugin.import_utils.device_operations.Cluster", + "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", + "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", + "netbox_librenms_plugin.import_utils.device_operations.Device", + "virtualization.models.VirtualMachine", + ] + + def setup_method(self): + self._patchers = [patch(p) for p in self.PATCHES] + mocks = [p.start() for p in self._patchers] + ( + self.mock_site_model, + self.mock_rack, + self.mock_cluster, + self.mock_role, + self.mock_match_type, + self.mock_find_platform, + self.mock_find_site, + self.mock_device, + self.mock_vm, + ) = mocks + + self.mock_find_site.return_value = { + "found": True, + "site": MagicMock(), + "match_type": "exact", + "confidence": 1.0, + } + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_rack.objects.filter.return_value = [] + self.mock_site_model.objects.all.return_value = [] + self.mock_vm.objects.filter.return_value.first.return_value = None + + def teardown_method(self): + for p in self._patchers: + p.stop() + + def _make_existing(self, librenms_id_value, serial="SN001"): + existing = MagicMock() + existing.name = "switch-01" + existing.serial = serial + existing.custom_field_data = {"librenms_id": librenms_id_value} + return existing + + def _setup_device_filter(self, existing): + def device_filter(*args, **kwargs): + result = MagicMock() + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + result.first.return_value = existing if q_has_librenms else None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + def test_legacy_int_sets_needs_migration_flag(self): + """Device with bare-integer librenms_id sets librenms_id_needs_migration=True.""" + existing = self._make_existing(librenms_id_value=42, serial="SN001") + self._setup_device_filter(existing) + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + result = validate_device_for_import( + {"device_id": 42, "hostname": "switch-01", "serial": "SN001"}, + include_vc_detection=False, + ) + + assert result["existing_match_type"] == "librenms_id" + assert result["librenms_id_needs_migration"] is True + assert result["serial_confirmed"] is True + + def test_legacy_int_no_serial_still_sets_flag(self): + """Legacy int format sets the migration flag even when serial is absent.""" + existing = self._make_existing(librenms_id_value=42, serial="") + self._setup_device_filter(existing) + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + result = validate_device_for_import( + {"device_id": 42, "hostname": "switch-01"}, + include_vc_detection=False, + ) + + assert result["librenms_id_needs_migration"] is True + assert result["serial_confirmed"] is False + + def test_json_format_does_not_set_flag(self): + """Device with JSON librenms_id does NOT set librenms_id_needs_migration.""" + existing = self._make_existing(librenms_id_value={"default": 42}, serial="SN001") + self._setup_device_filter(existing) + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + result = validate_device_for_import( + {"device_id": 42, "hostname": "switch-01", "serial": "SN001"}, + include_vc_detection=False, + ) + + assert result["existing_match_type"] == "librenms_id" + assert result["librenms_id_needs_migration"] is False + + def test_migrate_legacy_librenms_id_helper(self): + """migrate_legacy_librenms_id converts int to {server_key: int}.""" + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + obj = MagicMock() + obj.custom_field_data = {"librenms_id": 42} + result = migrate_legacy_librenms_id(obj, "primary") + + assert result is True + assert obj.custom_field_data["librenms_id"] == {"primary": 42} + + def test_migrate_legacy_librenms_id_noop_for_json(self): + """migrate_legacy_librenms_id is a no-op when value is already a dict.""" + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + obj = MagicMock() + obj.custom_field_data = {"librenms_id": {"primary": 42}} + result = migrate_legacy_librenms_id(obj, "primary") + + assert result is False + assert obj.custom_field_data["librenms_id"] == {"primary": 42} + + def test_migrate_legacy_librenms_id_noop_for_none(self): + """migrate_legacy_librenms_id is a no-op when librenms_id is absent.""" + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + obj = MagicMock() + obj.custom_field_data = {} + result = migrate_legacy_librenms_id(obj, "primary") + + assert result is False + + class TestDeviceConflictActionView: """Test DeviceConflictActionView conflict resolution actions.""" @@ -1668,6 +1813,7 @@ def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache) patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1708,6 +1854,7 @@ def test_update_action_sets_hostname_serial_and_librenms_id(self, mock_cache_key patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1744,6 +1891,7 @@ def test_update_serial_action_updates_serial_only(self, mock_cache_key, mock_cac patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1915,6 +2063,7 @@ def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cach patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1960,6 +2109,7 @@ def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cach patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -2036,6 +2186,7 @@ def test_sync_serial_action(self, mock_cache_key, mock_cache): patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() diff --git a/netbox_librenms_plugin/utils.py b/netbox_librenms_plugin/utils.py index 2c76b5494f..d30ccbba99 100644 --- a/netbox_librenms_plugin/utils.py +++ b/netbox_librenms_plugin/utils.py @@ -86,6 +86,39 @@ def find_by_librenms_id(model, librenms_id, server_key: str = "default"): ).first() +def migrate_legacy_librenms_id(obj, server_key: str = "default") -> bool: + """ + Migrate a legacy bare-integer ``librenms_id`` custom field to the JSON dict format, + scoped to *server_key*. + + Only performs the migration when the current value is a bare integer, i.e. a record + created before the multi-server JSON refactor. The integer is assumed to belong to + the server identified by *server_key* (the caller must verify this, e.g. by confirming + that the LibreNMS device ID and serial number both match). + + Does **not** call ``obj.save()`` — the caller is responsible for persisting the change. + + Args: + obj: NetBox object with a ``librenms_id`` custom field. + server_key: LibreNMS server key the legacy integer should be scoped to. + + Returns: + True if the value was migrated, False if it was already in the correct format. + """ + cf_value = obj.custom_field_data.get("librenms_id") + if not isinstance(cf_value, int): + return False + obj.custom_field_data["librenms_id"] = {server_key: cf_value} + logger.info( + "Migrated legacy librenms_id %d → {%r: %d} on %r", + cf_value, + server_key, + cf_value, + obj, + ) + return True + + def convert_speed_to_kbps(speed_bps: int) -> int: """ Convert speed from bits per second to kilobits per second. diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 8da9da8b0b..d5504dae52 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -901,12 +901,10 @@ def post(self, request, device_id): # Check for LibreNMS ID collision before any linking action if action in {"link", "update", "update_serial"}: - id_conflict = ( - Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)) - .exclude(pk=existing_device.pk) - .first() - ) - if id_conflict: + from netbox_librenms_plugin.utils import find_by_librenms_id + + id_conflict = find_by_librenms_id(Device, librenms_id, self.librenms_api.server_key) + if id_conflict and id_conflict.pk != existing_device.pk: return HttpResponse( f"LibreNMS ID conflict: ID {librenms_id} is already assigned to device " f"'{id_conflict.name}' (ID: {id_conflict.pk})", @@ -1061,6 +1059,30 @@ def post(self, request, device_id): else: return HttpResponse(f"No matching device type for '{hardware}'", status=400) + elif action == "migrate_librenms_id": + # Migrate legacy bare-integer librenms_id to the JSON dict format. + # Only safe when the integer matches the LibreNMS device ID for this server, + # confirmed by serial match (or explicit force). + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + cf_value = existing_device.custom_field_data.get("librenms_id") + if not isinstance(cf_value, int): + return HttpResponse( + "Device librenms_id is already in JSON format; no migration needed.", + status=400, + ) + if not validation.get("serial_confirmed") and not force: + return HttpResponse( + "Serial number not confirmed. Check the force checkbox to migrate without serial verification.", + status=400, + ) + migrate_legacy_librenms_id(existing_device, self.librenms_api.server_key) + existing_device.save() + logger.info( + f"Migrated legacy librenms_id on '{existing_device.name}' " + f"to {{{self.librenms_api.server_key!r}: {cf_value}}}" + ) + else: return HttpResponse(f"Unknown action: {action}", status=400) From 9be9e1542376bd590e3d16ff6bad2f18870c793f Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 00:00:16 +0100 Subject: [PATCH 16/39] =?UTF-8?q?fix:=20PR=20review=20fixes=20=E2=80=94=20?= =?UTF-8?q?schema,=20server=5Fkey,=20bulk=5Fimport,=20legacy=20badge,=20ac?= =?UTF-8?q?cessibility?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - device_operations.py: preserve device_type schema dict instead of replacing with raw dt_match; 'found' key is now always set regardless of match result - device_operations.py: use api.server_key instead of server_key or 'default' to consistently use the resolved server key after LibreNMSAPI instantiation - device_operations.py: move Q import to module level; remove inline 'from dcim.models import Rack' so test mock patching works correctly - bulk_import.py: fix 7 early-return sites to return ([], False) when return_cache_status=True, keeping the return shape consistent for callers - device_status.py: add 'Legacy ID' warning badge when librenms_id_needs_migration is set on an existing device matched by librenms_id - device_status.py: add aria-label to icon-only buttons for accessibility - device_validation_details.html: add rel='noopener noreferrer' to all target='_blank' anchors (8 links) - test_import_utils.py: add DeviceType to SERIAL_PATCHES so suggestions path is mocked; fix _create_view to use real instantiation; fix _setup_no_existing to use named variables instead of fragile negative indexing; remove dead mock_rack.objects.filter.return_value = [] lines (inline import bypass meant these were never effective); fix device_type assertions to use 'found' key --- .../import_utils/bulk_import.py | 14 +++--- .../import_utils/device_operations.py | 18 +++---- .../tables/device_status.py | 6 +++ .../htmx/device_validation_details.html | 16 +++--- .../tests/test_import_utils.py | 49 +++++++------------ 5 files changed, 45 insertions(+), 58 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index f910425226..5990d3d810 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -426,7 +426,7 @@ def process_device_filters( except (BrokenPipeError, ConnectionError, IOError) as e: if request: logger.info(f"Client disconnected during VC prefetch: {e}") - return [] + return ([], False) if return_cache_status else [] raise # Validate each device @@ -446,13 +446,13 @@ def process_device_filters( if rq_job.is_failed or rq_job.is_stopped: job.logger.warning("Job was already stopped before validation started") - return [] + return ([], False) if return_cache_status else [] except Exception: # Fall back to DB check if RQ check fails job.job.refresh_from_db() if job.job.status == JobStatusChoices.STATUS_FAILED: job.logger.warning("Job was stopped before validation started") - return [] + return ([], False) if return_cache_status else [] else: logger.info(f"Validating {total} devices") @@ -475,13 +475,13 @@ def process_device_filters( job.logger.info( f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." ) - return [] + return ([], False) if return_cache_status else [] except Exception: # If we can't check RQ status, fall back to DB status check job.job.refresh_from_db() if job.job.status == JobStatusChoices.STATUS_FAILED: job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") - return [] + return ([], False) if return_cache_status else [] elif request: # Check for client disconnect try: @@ -489,7 +489,7 @@ def process_device_filters( pass except (BrokenPipeError, ConnectionError, IOError): logger.info(f"Client disconnected during validation at device {idx}") - return [] + return ([], False) if return_cache_status else [] # Drop any cached validation/meta keys before recomputing device.pop("_validation", None) @@ -536,7 +536,7 @@ def process_device_filters( except (BrokenPipeError, ConnectionError, IOError) as e: if request: logger.info(f"Client disconnected during device validation: {e}") - return [] + return ([], False) if return_cache_status else [] raise # Set VC detection metadata diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index 161bd8e84b..d00b9297c4 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -5,6 +5,7 @@ from dcim.models import Device, DeviceRole, DeviceType, Rack, Site from django.core.cache import cache from django.db import transaction +from django.db.models import Q from django.utils import timezone from virtualization.models import Cluster # noqa: F401 — used by test mock.patch targets @@ -483,7 +484,10 @@ def validate_device_for_import( if chassis_match and chassis_match["matched"]: dt_match = chassis_match - result["device_type"] = dt_match + # Update result keys individually to preserve the existing schema (especially "found") + result["device_type"]["found"] = dt_match["matched"] + result["device_type"]["device_type"] = dt_match.get("device_type") + result["device_type"]["match_type"] = dt_match.get("match_type") if not dt_match["matched"]: result["device_type"]["found"] = False @@ -498,11 +502,6 @@ def validate_device_for_import( } for dt in all_device_types ] - else: - # Rename 'matched' to 'found' for consistency - result["device_type"]["found"] = dt_match["matched"] - result["device_type"]["device_type"] = dt_match["device_type"] - result["device_type"]["match_type"] = dt_match["match_type"] # 4. DeviceRole (required) - Must be manually selected by user logger.debug(f"[{hostname}] Issues BEFORE adding role issue: {result['issues']}") @@ -527,9 +526,6 @@ def validate_device_for_import( available_racks = cache.get(cache_key) if available_racks is None: - from dcim.models import Rack - from django.db.models import Q - # Query racks for this site - include both: # 1. Racks assigned to locations within the site # 2. Racks directly assigned to the site (without location) @@ -701,7 +697,7 @@ def import_single_device( libre_device, use_sysname=use_sysname_opt, strip_domain=strip_domain_opt, - server_key=server_key or "default", + server_key=api.server_key, ) # Check if device already exists @@ -787,7 +783,7 @@ def import_single_device( "role": device_role, "status": "active" if libre_device.get("status") == 1 else "offline", "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - "custom_field_data": {"librenms_id": {(server_key or "default"): int(device_id)}}, + "custom_field_data": {"librenms_id": {api.server_key: int(device_id)}}, } # Add optional fields diff --git a/netbox_librenms_plugin/tables/device_status.py b/netbox_librenms_plugin/tables/device_status.py index 3a548bcd96..58eeb7c2ad 100644 --- a/netbox_librenms_plugin/tables/device_status.py +++ b/netbox_librenms_plugin/tables/device_status.py @@ -479,15 +479,21 @@ def render_actions(self, value, record): btn_class = "btn-outline-warning" btn_icon = "mdi-information-outline" btn_label = " Details" + elif match_type == "librenms_id" and validation.get("librenms_id_needs_migration"): + btn_class = "btn-outline-warning" + btn_icon = "mdi-database-alert" + btn_label = " Legacy ID" else: btn_class = "btn-outline-success" btn_icon = "mdi-check-circle" btn_label = "" btn_title = "Resolve conflict" if (has_actions or has_mismatch) else "View details" + aria_attr = f'aria-label="{btn_title}" ' if btn_label == "" else "" buttons.append( f'
@@ -479,7 +479,7 @@
Device Information
{% endif %} — Exists as - {{ validation.existing_device.name }}, + {{ validation.existing_device.name }}, not linked to LibreNMS.
@@ -517,7 +517,7 @@
Device Information
IP match — Device with IP {{ libre_device.ip }} exists as - {{ validation.existing_device.name }}. + {{ validation.existing_device.name }}. Consider adding LibreNMS ID manually.
@@ -526,7 +526,7 @@
Device Information
Exists — Device already exists as - {{ validation.existing_device.name }}. + {{ validation.existing_device.name }}.
{% endif %} @@ -569,17 +569,17 @@
Device Information
{% if validation.existing_device %} {% if validation.import_as_vm or validation.existing_device.cluster %} + class="btn btn-primary btn-sm" target="_blank" rel="noopener noreferrer"> View VM in NetBox {% else %} + class="btn btn-primary btn-sm" target="_blank" rel="noopener noreferrer"> View in NetBox {% if validation.existing_match_type == 'librenms_id' %} + class="btn btn-outline-primary btn-sm" target="_blank" rel="noopener noreferrer"> Full Sync Page {% endif %} diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 1b8df20727..b2c992be74 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -341,7 +341,6 @@ def test_validate_device_site_match_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [mock_site] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -399,7 +398,6 @@ def test_validate_device_site_not_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -461,7 +459,6 @@ def test_validate_device_platform_match_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -519,7 +516,6 @@ def test_validate_device_platform_not_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -577,7 +573,6 @@ def test_validate_device_type_match_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -638,7 +633,6 @@ def test_validate_device_type_not_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -651,7 +645,7 @@ def test_validate_device_type_not_found( result = validate_device_for_import(device_data, include_vc_detection=False) - assert result["device_type"]["matched"] is False + assert result["device_type"]["found"] is False assert any("device type" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") @@ -698,7 +692,6 @@ def test_validate_device_role_required( } mock_role.objects.all.return_value = [MagicMock(id=1, name="Access Switch")] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [mock_site] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -757,7 +750,6 @@ def test_validate_device_handles_empty_location( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -816,7 +808,6 @@ def test_validate_device_handles_empty_os( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -877,7 +868,6 @@ def test_validate_device_handles_empty_hardware( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -891,7 +881,7 @@ def test_validate_device_handles_empty_hardware( result = validate_device_for_import(device_data, include_vc_detection=False) assert result is not None - assert result["device_type"]["matched"] is False + assert result["device_type"]["found"] is False @patch("virtualization.models.VirtualMachine") @patch("netbox_librenms_plugin.import_utils.device_operations.Device") @@ -974,7 +964,6 @@ def test_validate_device_returns_complete_state( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1045,7 +1034,6 @@ def test_validate_device_import_as_vm( mock_clusters = [MagicMock(id=1, name="VMware Cluster")] mock_cluster.objects.all.return_value = mock_clusters mock_cache.get.return_value = None # Force cache miss to trigger Cluster.objects.all() - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1109,6 +1097,7 @@ class TestSerialNumberMatching: "netbox_librenms_plugin.import_utils.device_operations.Rack", "netbox_librenms_plugin.import_utils.device_operations.Cluster", "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", + "netbox_librenms_plugin.import_utils.device_operations.DeviceType", "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", @@ -1125,12 +1114,14 @@ def _start_patches(self): self.mock_rack, self.mock_cluster, self.mock_role, + self.mock_device_type, self.mock_match_type, self.mock_find_platform, self.mock_find_site, self.mock_device, self.mock_vm, ) = mocks + self.mock_device_type.objects.all.return_value = [] def _stop_patches(self): """Stop all patches.""" @@ -1265,7 +1256,6 @@ def _setup_no_match_mocks(self): self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] def test_serial_dash_ignored(self): @@ -1366,7 +1356,6 @@ def device_filter(*args, **kwargs): self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1412,7 +1401,6 @@ def device_filter(*args, **kwargs): self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1451,7 +1439,6 @@ def device_filter(*args, **kwargs): self.mock_match_type.return_value = {"matched": True, "device_type": mock_dt, "match_type": "exact"} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1492,7 +1479,6 @@ def device_filter(**kwargs): self.mock_match_type.return_value = {"matched": True, "device_type": MagicMock(), "match_type": "exact"} self.mock_role.objects.all.return_value = [mock_existing_role] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: @@ -1548,7 +1534,6 @@ def device_filter(**kwargs): } self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: @@ -1594,7 +1579,6 @@ def device_filter(**kwargs): self.mock_match_type.return_value = {"matched": True, "device_type": same_device_type, "match_type": "exact"} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: @@ -1654,7 +1638,6 @@ def setup_method(self): self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] self.mock_vm.objects.filter.return_value.first.return_value = None @@ -1766,7 +1749,7 @@ def _create_view(self): """Create a DeviceConflictActionView instance with mocked dependencies.""" from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView - view = object.__new__(DeviceConflictActionView) + view = DeviceConflictActionView() view._librenms_api = MagicMock() view._librenms_api.server_key = "default" view.request = MagicMock() @@ -2363,14 +2346,17 @@ class TestDeviceNamingPreferences: def _setup_no_existing(self, mocks): """Configure mocks so no existing device is found.""" - mock_vm = mocks[-1] # VirtualMachine - mock_device = mocks[-2] # Device - mock_find_site = mocks[-3] - mock_find_platform = mocks[-4] - mock_match_type = mocks[-5] - mock_role = mocks[-6] - mock_rack = mocks[-8] - mock_site_model = mocks[-9] + ( + mock_site_model, + mock_rack, + mock_cluster, + mock_role, + mock_match_type, + mock_find_platform, + mock_find_site, + mock_device, + mock_vm, + ) = mocks mock_vm.objects.filter.return_value.first.return_value = None mock_device.objects.filter.return_value.first.return_value = None @@ -2391,7 +2377,6 @@ def _setup_no_existing(self, mocks): "match_type": None, } mock_role.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] @patch("virtualization.models.VirtualMachine") From 46d3775df1c3d2b2e5503819429fab2decb482d0 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 00:03:28 +0100 Subject: [PATCH 17/39] fix: update test_init to expect json CF type instead of integer --- netbox_librenms_plugin/tests/test_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox_librenms_plugin/tests/test_init.py b/netbox_librenms_plugin/tests/test_init.py index 6225484dd6..8090708b89 100644 --- a/netbox_librenms_plugin/tests/test_init.py +++ b/netbox_librenms_plugin/tests/test_init.py @@ -46,7 +46,7 @@ def test_creates_custom_field_when_missing( MockCustomField.objects.get_or_create.assert_called_once_with( name="librenms_id", defaults={ - "type": "integer", + "type": "json", "label": "LibreNMS ID", "description": "LibreNMS Device ID for synchronization (auto-created by plugin)", "required": False, From 4c31e0b21bd336a1cf045253adc82e464ddab733 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 00:26:28 +0100 Subject: [PATCH 18/39] PR review: code quality, accessibility and correctness fixes - import_utils/cache.py: guard datetime.fromisoformat with try/except fallback; replace hash() with deterministic sha256 for cross-process stable cache keys - import_utils/filters.py: replace hash() with deterministic sha256 for librenms_devices_import cache key; add hashlib/json imports - import_utils/permissions.py: remove unused logger and logging import - import_utils/virtual_chassis.py: fix blank serial skip logic - only skip member when serial is non-empty AND matches master serial; same fix in expected_members count - device_validation_details.html: add aria-label to 6 icon-only sync buttons for screen-reader accessibility - test_import_utils.py: replace fragile mocks[-2] indexing with named unpacking - views/imports/actions.py: import escape(); HTML-escape untrusted values (id_conflict.name, librenms_os, hardware, action) in HttpResponse f-strings - views/sync/interfaces.py: guard handle_mac_address with is_device_interface to avoid setting MAC on VMInterface objects - import_utils/device_operations.py: reassign import_as_vm from result dict after existing-object detection so downstream branches use effective VM mode --- netbox_librenms_plugin/import_utils/cache.py | 14 +++++++++++--- .../import_utils/device_operations.py | 4 ++++ netbox_librenms_plugin/import_utils/filters.py | 11 +++++++++-- netbox_librenms_plugin/import_utils/permissions.py | 4 ---- .../import_utils/virtual_chassis.py | 8 +++++--- .../htmx/device_validation_details.html | 12 ++++++------ netbox_librenms_plugin/tests/test_import_utils.py | 13 ++++++++++++- netbox_librenms_plugin/views/imports/actions.py | 9 +++++---- netbox_librenms_plugin/views/sync/interfaces.py | 2 +- 9 files changed, 53 insertions(+), 24 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/cache.py b/netbox_librenms_plugin/import_utils/cache.py index 716e9dc6a1..7430d129f4 100644 --- a/netbox_librenms_plugin/import_utils/cache.py +++ b/netbox_librenms_plugin/import_utils/cache.py @@ -1,5 +1,7 @@ """Cache key generation and management for device import operations.""" +import hashlib +import json import logging from django.core.cache import cache @@ -71,9 +73,15 @@ def get_active_cached_searches(server_key: str) -> list[dict]: metadata = cache.get(cache_key) if metadata: # Cache still exists, calculate time remaining - cached_at = datetime.fromisoformat(metadata.get("cached_at")) cache_timeout = metadata.get("cache_timeout", 300) now = datetime.now(timezone.utc) + try: + cached_at_raw = metadata.get("cached_at") + cached_at = ( + datetime.fromisoformat(cached_at_raw) if cached_at_raw else datetime.fromtimestamp(0, timezone.utc) + ) + except (ValueError, TypeError): + cached_at = datetime.fromtimestamp(0, timezone.utc) age_seconds = (now - cached_at).total_seconds() remaining_seconds = max(0, cache_timeout - age_seconds) @@ -130,8 +138,8 @@ def get_validated_device_cache_key(server_key: str, filters: dict, device_id: in >>> key 'validated_device_default_-1234567890_123_vc' """ - # Sort filters for consistent hashing - filter_hash = hash(str(sorted(filters.items()))) + # Sort filters for a deterministic, cross-process stable hash + filter_hash = hashlib.sha256(json.dumps(sorted(filters.items()), sort_keys=True).encode()).hexdigest()[:16] vc_part = "vc" if vc_enabled else "novc" return f"validated_device_{server_key}_{filter_hash}_{device_id}_{vc_part}" diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index d00b9297c4..4d1d5bcb03 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -438,6 +438,10 @@ def validate_device_for_import( ) result["can_import"] = False + # Refresh local variable to reflect any VM-mode adjustments made during detection + # (e.g. existing VM found by hostname sets result["import_as_vm"] = True) + import_as_vm = result["import_as_vm"] + # Validate based on import type (Device or VM) if import_as_vm: # 2. For VMs: Validate Cluster (required) - Must be manually selected diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py index 27f4449266..9f55576a26 100644 --- a/netbox_librenms_plugin/import_utils/filters.py +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -1,5 +1,7 @@ """Device filtering and retrieval from LibreNMS.""" +import hashlib +import json import logging from typing import List @@ -170,8 +172,13 @@ def get_librenms_devices_for_import( # We'll filter client-side if needed # Use caching to avoid repeated API calls - # Include both API and client filters in cache key - cache_key = f"librenms_devices_import_{server_key}_{hash(str(api_filters))}_{hash(str(client_filters))}" + # Include both API and client filters in cache key (deterministic, cross-process stable) + def _hash(d): + return hashlib.sha256( + json.dumps(sorted(d.items()) if isinstance(d, dict) else d, sort_keys=True).encode() + ).hexdigest()[:16] + + cache_key = f"librenms_devices_import_{server_key}_{_hash(api_filters)}_{_hash(client_filters)}" from_cache = False if force_refresh: diff --git a/netbox_librenms_plugin/import_utils/permissions.py b/netbox_librenms_plugin/import_utils/permissions.py index 9e9b4521e0..742e50893c 100644 --- a/netbox_librenms_plugin/import_utils/permissions.py +++ b/netbox_librenms_plugin/import_utils/permissions.py @@ -1,11 +1,7 @@ """Permission check helpers for device import operations.""" -import logging - from django.core.exceptions import PermissionDenied -logger = logging.getLogger(__name__) - def check_user_permissions(user, permissions): """ diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py index 79db61e3fe..4dc44e4182 100644 --- a/netbox_librenms_plugin/import_utils/virtual_chassis.py +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -373,8 +373,8 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis members_created = 0 for member in members_info: - # Skip if this is the master's serial - if member.get("serial") == master_device.serial: + # Skip if this is the master's serial (only when both serials are non-empty) + if member.get("serial") and member.get("serial") == master_device.serial: continue serial = member.get("serial") @@ -414,7 +414,9 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis position += 1 # Validate member count - expected_members = len([m for m in members_info if m.get("serial") != master_device.serial]) + expected_members = len( + [m for m in members_info if not (m.get("serial") and m.get("serial") == master_device.serial)] + ) if members_created < expected_members: logger.warning( f"Created {members_created} members but expected {expected_members}. " diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html index 21984b37b0..5e30e039f2 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html @@ -108,7 +108,7 @@
Device Information
{% csrf_token %} - @@ -156,7 +156,7 @@
Device Information
- @@ -169,7 +169,7 @@
Device Information
{% csrf_token %} - @@ -209,7 +209,7 @@
Device Information
{% csrf_token %} - @@ -263,7 +263,7 @@
Device Information
{% csrf_token %} - @@ -287,7 +287,7 @@
Device Information
{% csrf_token %} - diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index b2c992be74..27c157b1dd 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -2466,7 +2466,18 @@ def test_duplicate_detection_uses_resolved_name(self, *mocks): """Duplicate detection should match against the resolved name, not raw hostname.""" self._setup_no_existing(mocks) - mock_device = mocks[-2] # Device + # Unpack using same order as _setup_no_existing / @patch decorators (bottom-up) + ( + _mock_site, + _mock_rack, + _mock_cluster, + _mock_role, + _mock_hw, + _mock_platform, + _mock_find_site, + mock_device, + _mock_vm, + ) = mocks existing = MagicMock() existing.name = "core-switch" existing.serial = "" diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index d5504dae52..21682e6403 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -8,6 +8,7 @@ from django.core.exceptions import PermissionDenied from django.http import HttpResponse, JsonResponse from django.shortcuts import redirect, render +from django.utils.html import escape from django.views import View from netbox_librenms_plugin.import_utils import ( @@ -907,7 +908,7 @@ def post(self, request, device_id): if id_conflict and id_conflict.pk != existing_device.pk: return HttpResponse( f"LibreNMS ID conflict: ID {librenms_id} is already assigned to device " - f"'{id_conflict.name}' (ID: {id_conflict.pk})", + f"'{escape(id_conflict.name)}' (ID: {id_conflict.pk})", status=409, ) @@ -1042,7 +1043,7 @@ def post(self, request, device_id): existing_device.save() logger.info(f"Synced platform on '{existing_device.name}' to {match_result['platform']}") else: - return HttpResponse(f"Platform '{librenms_os}' not found in NetBox", status=400) + return HttpResponse(f"Platform '{escape(librenms_os)}' not found in NetBox", status=400) else: return HttpResponse("No OS info from LibreNMS", status=400) @@ -1057,7 +1058,7 @@ def post(self, request, device_id): existing_device.save() logger.info(f"Synced device type on '{existing_device.name}' to {hw_match['device_type']}") else: - return HttpResponse(f"No matching device type for '{hardware}'", status=400) + return HttpResponse(f"No matching device type for '{escape(hardware)}'", status=400) elif action == "migrate_librenms_id": # Migrate legacy bare-integer librenms_id to the JSON dict format. @@ -1084,7 +1085,7 @@ def post(self, request, device_id): ) else: - return HttpResponse(f"Unknown action: {action}", status=400) + return HttpResponse(f"Unknown action: {escape(action)}", status=400) # Clear cached validation so re-validation picks up the changes cache_key = get_import_device_cache_key(device_id, self.librenms_api.server_key) diff --git a/netbox_librenms_plugin/views/sync/interfaces.py b/netbox_librenms_plugin/views/sync/interfaces.py index 1bc5babf02..89bbc5ac92 100644 --- a/netbox_librenms_plugin/views/sync/interfaces.py +++ b/netbox_librenms_plugin/views/sync/interfaces.py @@ -241,7 +241,7 @@ def update_interface_attributes( else (admin_status.lower() == "up" if isinstance(admin_status, str) else bool(admin_status)) ) - if "mac_address" not in exclude_columns: + if "mac_address" not in exclude_columns and is_device_interface: ifPhysAddress = librenms_interface.get("ifPhysAddress") self.handle_mac_address(interface, ifPhysAddress) From 54b7623fb6fccf89bdc8dd362859dbdb2beb9e8c Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 00:27:27 +0100 Subject: [PATCH 19/39] PR review: inventory-specific fixes - __init__.py: move _executed = True to inside try block after successful custom field creation, so failures allow retry on next post_migrate - tests/test_init.py: assert _executed is False after exception in test_exception_does_not_propagate to verify retry behaviour - views/base/librenms_sync_view.py: remove redundant found_in_librenms = True inside mismatched_device block (already set on line 184 after API call) --- netbox_librenms_plugin/__init__.py | 4 +++- netbox_librenms_plugin/tests/test_init.py | 3 +++ netbox_librenms_plugin/views/base/librenms_sync_view.py | 1 - 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/netbox_librenms_plugin/__init__.py b/netbox_librenms_plugin/__init__.py index dc933681a1..206a3009c8 100644 --- a/netbox_librenms_plugin/__init__.py +++ b/netbox_librenms_plugin/__init__.py @@ -81,7 +81,6 @@ def _ensure_librenms_id_custom_field(sender, **kwargs): # Only run once per migrate invocation (post_migrate fires per-app). if getattr(_ensure_librenms_id_custom_field, "_executed", False): return - _ensure_librenms_id_custom_field._executed = True # not reset; see comment above import logging @@ -128,6 +127,9 @@ def _ensure_librenms_id_custom_field(sender, **kwargs): logging.getLogger("netbox_librenms_plugin").info( "Auto-created 'librenms_id' custom field for Device, VirtualMachine, Interface, VMInterface" ) + + # Only mark as executed after successful completion to allow retry on failure. + _ensure_librenms_id_custom_field._executed = True except Exception as e: # Don't break startup if custom field creation fails (e.g., during initial migration), # but log the error so it's not silently swallowed. diff --git a/netbox_librenms_plugin/tests/test_init.py b/netbox_librenms_plugin/tests/test_init.py index 8090708b89..426f6736c0 100644 --- a/netbox_librenms_plugin/tests/test_init.py +++ b/netbox_librenms_plugin/tests/test_init.py @@ -142,6 +142,9 @@ def test_exception_does_not_propagate(self, MockCustomField): call_args = logger_instance.exception.call_args assert "librenms_id" in call_args[0][0] + # On failure, _executed must NOT be set — failed attempts should allow retry + assert not getattr(_ensure_librenms_id_custom_field, "_executed", False) + @patch("dcim.models.Interface", new_callable=MagicMock) @patch("dcim.models.Device", new_callable=MagicMock) @patch("virtualization.models.VMInterface", new_callable=MagicMock) diff --git a/netbox_librenms_plugin/views/base/librenms_sync_view.py b/netbox_librenms_plugin/views/base/librenms_sync_view.py index 4a85cb1642..8ccd4304c6 100644 --- a/netbox_librenms_plugin/views/base/librenms_sync_view.py +++ b/netbox_librenms_plugin/views/base/librenms_sync_view.py @@ -233,7 +233,6 @@ def get_librenms_device_info(self, obj): mismatched_device = False else: # Device is still found (we have librenms_id), just mismatched - found_in_librenms = True mismatched_device = True librenms_device_details["netbox_dns_name"] = netbox_dns_name or "-" From 0686cc79f8cb740ba4396f9fbea26bf9663a7a6d Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 09:26:03 +0100 Subject: [PATCH 20/39] Fix _refresh_existing_device: readiness logic and server_key - Recompute can_import/is_ready when cached existing_device is gone to match validate_device_for_import: can_import = not issues; VMs only require cluster, devices require site+device_type+device_role - Remove bare can_import=True which ignored cached issue list - Add server_key parameter to _refresh_existing_device so librenms_id lookup uses the correct server key instead of None (_server_key was never set on device dicts, causing JSON-format lookups to miss records) --- .../import_utils/bulk_import.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 5990d3d810..4d28ebc17a 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -268,7 +268,7 @@ def bulk_import_devices( ) -def _refresh_existing_device(validation: dict, libre_device: dict = None) -> None: +def _refresh_existing_device(validation: dict, libre_device: dict = None, server_key: str = "default") -> None: """Refresh existing_device from DB to pick up changes made in NetBox since caching. When existing_device is None (wasn't found at cache time), re-check if the device @@ -290,20 +290,23 @@ def _refresh_existing_device(validation: dict, libre_device: dict = None) -> Non if hasattr(refreshed, "role") and refreshed.role: validation["device_role"] = {"found": True, "role": refreshed.role} else: - # Device was deleted since caching — recompute readiness + # Device was deleted since caching — recompute readiness to match + # validate_device_for_import logic. validation["existing_device"] = None validation["existing_match_type"] = None - validation["can_import"] = True + can_import = not bool(validation.get("issues")) if validation.get("import_as_vm"): - validation["is_ready"] = bool( - validation.get("site", {}).get("found") and validation.get("device_role", {}).get("found") - ) + # VMs only require a cluster (site/role not mandatory) + is_ready = can_import and bool(validation.get("cluster", {}).get("found")) else: - validation["is_ready"] = bool( - validation.get("site", {}).get("found") - and validation.get("device_type", {}).get("found") - and validation.get("device_role", {}).get("found") + is_ready = ( + can_import + and bool(validation.get("site", {}).get("found")) + and bool(validation.get("device_type", {}).get("found")) + and bool(validation.get("device_role", {}).get("found")) ) + validation["can_import"] = can_import + validation["is_ready"] = is_ready except Exception as e: existing_id = getattr(existing, "pk", "unknown") if existing else "none" logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") @@ -318,7 +321,6 @@ def _refresh_existing_device(validation: dict, libre_device: dict = None) -> Non librenms_id = libre_device.get("device_id") hostname = libre_device.get("hostname", "") sys_name = libre_device.get("sysName", "") - server_key = libre_device.get("_server_key") new_device = None match_type = None @@ -513,7 +515,7 @@ def process_device_filters( # Refresh existing_device from DB to avoid stale data # (user may have changed role, name, etc. in NetBox) - _refresh_existing_device(device["_validation"], libre_device=device) + _refresh_existing_device(device["_validation"], libre_device=device, server_key=api.server_key) # Apply exclude_existing filter if enabled if exclude_existing: From 9a13a27c2e378f84c4cf4f97ea9f9c8ef314ae6d Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 09:32:13 +0100 Subject: [PATCH 21/39] chore: uv.lock --- uv.lock | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 uv.lock diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000000..4f6c48c6f1 --- /dev/null +++ b/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.12.0" + +[[package]] +name = "netbox-librenms-plugin" +version = "0.4.2" +source = { editable = "." } From 0234f2e7ee4502bc40bca645d044dc5418161d5d Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 10:13:55 +0100 Subject: [PATCH 22/39] fix: batch 4 PR review fixes - virtual_chassis.py: load VC member name pattern once before loops to avoid N DB queries per member creation/update - virtual_chassis.py: fix parent_index check (0 is valid, use 'is None') - actions.py: use _resolve_naming_preferences() in link/update/sync_name actions instead of reading POST toggles directly (respects user prefs) - actions.py: HTML-escape incoming_serial and conflict_device.name in all serial-conflict HttpResponse messages (XSS fix) - actions.py: verify existing_device_id matches validated conflict target before mutation (only enforced when validation has existing_device set) - actions.py: add comment explaining collision check is functionally correct - bulk_import.py: remove dead wsgi.input try/except (no-op, cannot catch BrokenPipeError as written) - cache.py: fix filter_parts to use 'if v is not None' (preserves 0/False) - cache.py: normalize naive datetime to UTC before age_seconds subtraction - cache.py: update docstring example to reflect SHA-256 hex prefix format - filters.py: use api.server_key (always resolved) in cache key construction - device_operations.py: add comments explaining direct CF access is needed for legacy integer format detection (not a bug) - test_import_utils.py: add comment explaining dcim.models.Platform patch is correct (inline import at call time) - test_import_utils.py: fix _create_request to always include both toggle keys to prevent DB access in _resolve_naming_preferences fallback --- .../import_utils/bulk_import.py | 8 --- netbox_librenms_plugin/import_utils/cache.py | 10 ++- .../import_utils/device_operations.py | 10 ++- .../import_utils/filters.py | 5 +- .../import_utils/virtual_chassis.py | 41 ++++++++---- .../tests/test_import_utils.py | 16 +++-- .../views/imports/actions.py | 66 +++++++++---------- 7 files changed, 88 insertions(+), 68 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 4d28ebc17a..62046941ae 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -484,14 +484,6 @@ def process_device_filters( if job.job.status == JobStatusChoices.STATUS_FAILED: job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") return ([], False) if return_cache_status else [] - elif request: - # Check for client disconnect - try: - if hasattr(request, "META") and request.META.get("wsgi.input"): - pass - except (BrokenPipeError, ConnectionError, IOError): - logger.info(f"Client disconnected during validation at device {idx}") - return ([], False) if return_cache_status else [] # Drop any cached validation/meta keys before recomputing device.pop("_validation", None) diff --git a/netbox_librenms_plugin/import_utils/cache.py b/netbox_librenms_plugin/import_utils/cache.py index 7430d129f4..03f711ad02 100644 --- a/netbox_librenms_plugin/import_utils/cache.py +++ b/netbox_librenms_plugin/import_utils/cache.py @@ -21,8 +21,9 @@ def get_cache_metadata_key(server_key: str, filters: dict, vc_enabled: bool) -> Returns: str: Consistent cache key for metadata """ - # Sort filter items to ensure consistent key generation - filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v) + # Sort filter items to ensure consistent key generation; use "is not None" to preserve + # valid falsy values like 0 and False (filtering only None/missing entries). + filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v is not None) return f"librenms_filter_cache_metadata_{server_key}_{filter_parts}_{vc_enabled}" @@ -80,6 +81,9 @@ def get_active_cached_searches(server_key: str) -> list[dict]: cached_at = ( datetime.fromisoformat(cached_at_raw) if cached_at_raw else datetime.fromtimestamp(0, timezone.utc) ) + # Normalize naive datetimes (e.g., stored without tzinfo) to UTC + if cached_at.tzinfo is None: + cached_at = cached_at.replace(tzinfo=timezone.utc) except (ValueError, TypeError): cached_at = datetime.fromtimestamp(0, timezone.utc) age_seconds = (now - cached_at).total_seconds() @@ -136,7 +140,7 @@ def get_validated_device_cache_key(server_key: str, filters: dict, device_id: in Example: >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) >>> key - 'validated_device_default_-1234567890_123_vc' + 'validated_device_default_-e3b0c44298fc1c14_123_vc' """ # Sort filters for a deterministic, cross-process stable hash filter_hash = hashlib.sha256(json.dumps(sorted(filters.items()), sort_keys=True).encode()).hexdigest()[:16] diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index 4d1d5bcb03..b0a3043686 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -272,7 +272,10 @@ def validate_device_for_import( result["import_as_vm"] = True # Force VM mode since VM exists result["can_import"] = False - # Detect legacy bare-integer format so UI can offer a migration action + # Detect legacy bare-integer format so UI can offer a migration action. + # Direct access needed to detect legacy integer format for migration prompt: + # LibreNMSAPI.get_librenms_id() returns an int in both formats, so only the + # raw type check on custom_field_data reveals whether migration is needed. if isinstance(existing_vm.custom_field_data.get("librenms_id"), int): result["librenms_id_needs_migration"] = True @@ -299,7 +302,10 @@ def validate_device_for_import( result["existing_match_type"] = "librenms_id" result["can_import"] = False - # Detect legacy bare-integer format so UI can offer a migration action + # Detect legacy bare-integer format so UI can offer a migration action. + # Direct access needed to detect legacy integer format for migration prompt: + # LibreNMSAPI.get_librenms_id() returns an int in both formats, so only the + # raw type check on custom_field_data reveals whether migration is needed. if isinstance(existing_device.custom_field_data.get("librenms_id"), int): result["librenms_id_needs_migration"] = True diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py index 9f55576a26..4085dfa325 100644 --- a/netbox_librenms_plugin/import_utils/filters.py +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -172,13 +172,14 @@ def get_librenms_devices_for_import( # We'll filter client-side if needed # Use caching to avoid repeated API calls - # Include both API and client filters in cache key (deterministic, cross-process stable) + # Include both API and client filters in cache key (deterministic, cross-process stable). + # Use api.server_key (always resolved) rather than the raw server_key arg (may differ). def _hash(d): return hashlib.sha256( json.dumps(sorted(d.items()) if isinstance(d, dict) else d, sort_keys=True).encode() ).hexdigest()[:16] - cache_key = f"librenms_devices_import_{server_key}_{_hash(api_filters)}_{_hash(client_filters)}" + cache_key = f"librenms_devices_import_{api.server_key}_{_hash(api_filters)}_{_hash(client_filters)}" from_cache = False if force_refresh: diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py index 4dc44e4182..7f775faa79 100644 --- a/netbox_librenms_plugin/import_utils/virtual_chassis.py +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -175,7 +175,7 @@ def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> d logger.debug(f"VC detection: Found parent container at index {parent_index} for device {device_id}") break - if not parent_index: + if parent_index is None: return None # Step 3: Get children chassis at next level @@ -232,7 +232,19 @@ def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> d return None -def _generate_vc_member_name(master_name: str, position: int, serial: str = None) -> str: +def _load_vc_member_name_pattern() -> str: + """Load the VC member name pattern from settings, with fallback to default.""" + from ..models import LibreNMSSettings + + try: + settings = LibreNMSSettings.objects.first() + return settings.vc_member_name_pattern if settings else "-M{position}" + except Exception as e: + logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") + return "-M{position}" + + +def _generate_vc_member_name(master_name: str, position: int, serial: str = None, pattern: str = None) -> str: """ Generate name for VC member device using configured pattern from settings. @@ -240,6 +252,9 @@ def _generate_vc_member_name(master_name: str, position: int, serial: str = None master_name: Name of the master/primary device position: VC position number serial: Optional serial number of the member device + pattern: Optional pre-loaded name pattern; if None, loaded from settings. + Pass a pre-loaded pattern when calling inside a loop to avoid + repeated DB queries. Returns: Generated member device name @@ -250,16 +265,8 @@ def _generate_vc_member_name(master_name: str, position: int, serial: str = None pattern="-SW{position}" -> "switch01-SW2" pattern=" [{serial}]" -> "switch01 [ABC123]" """ - # Import here to avoid circular dependency - from ..models import LibreNMSSettings - - # Get pattern from settings with fallback to default - try: - settings = LibreNMSSettings.objects.first() - pattern = settings.vc_member_name_pattern if settings else "-M{position}" - except Exception as e: - logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") - pattern = "-M{position}" + if pattern is None: + pattern = _load_vc_member_name_pattern() # Prepare format variables format_vars = { @@ -294,6 +301,8 @@ def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: if not vc_data or not vc_data.get("is_stack"): return vc_data + # Load naming pattern once to avoid a DB query per member + vc_pattern = _load_vc_member_name_pattern() for idx, member in enumerate(vc_data.get("members", [])): raw_position = member.get("position", idx) try: @@ -302,7 +311,9 @@ def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: base_position = idx position = base_position + 1 # Convert to 1-based position member["position"] = base_position - member["suggested_name"] = _generate_vc_member_name(master_name, position, serial=member.get("serial")) + member["suggested_name"] = _generate_vc_member_name( + master_name, position, serial=member.get("serial"), pattern=vc_pattern + ) return vc_data @@ -371,6 +382,8 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis # Create member devices for remaining positions position = 2 # Start at 2 (master is 1) members_created = 0 + # Load naming pattern once to avoid a DB query per member + vc_pattern = _load_vc_member_name_pattern() for member in members_info: # Skip if this is the master's serial (only when both serials are non-empty) @@ -389,7 +402,7 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") continue - member_name = _generate_vc_member_name(master_base_name, position, serial=serial) + member_name = _generate_vc_member_name(master_base_name, position, serial=serial, pattern=vc_pattern) # Check for duplicate name if Device.objects.filter(name=member_name).exists(): diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 27c157b1dd..ba88025fc3 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1759,11 +1759,14 @@ def _create_view(self): def _create_request(self, action, existing_device_id, use_sysname=False, strip_domain=False): """Create a mock request with POST data.""" request = MagicMock() - post_data = {"action": action, "existing_device_id": str(existing_device_id)} - if use_sysname: - post_data["use-sysname-toggle"] = "on" - if strip_domain: - post_data["strip-domain-toggle"] = "on" + # Always include both toggles so _resolve_naming_preferences never falls through + # to the user-pref/settings DB path, which would hit the real database. + post_data = { + "action": action, + "existing_device_id": str(existing_device_id), + "use-sysname-toggle": "on" if use_sysname else "off", + "strip-domain-toggle": "on" if strip_domain else "off", + } request.POST = post_data return request @@ -2199,6 +2202,9 @@ def test_sync_platform_action(self, mock_cache_key, mock_cache): patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + # Patch at dcim.models level: find_matching_platform uses an inline + # 'from dcim.models import Platform' so patching dcim.models.Platform + # correctly intercepts the binding at call time. patch("dcim.models.Platform") as mock_platform_cls, ): mock_device_cls.objects.get.return_value = existing_device diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 21682e6403..41e19b669f 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -880,6 +880,13 @@ def post(self, request, device_id): if not libre_device: return HttpResponse("LibreNMS device not found", status=404) + # Verify the POSTed existing_device_id matches the validated conflict target. + # Without this check, an attacker could mutate an arbitrary device. + # Only enforce when validation has a known existing_device (conflict case). + validated_existing = validation.get("existing_device") if validation else None + if validated_existing is not None and validated_existing.pk != existing_device.pk: + return HttpResponse("Device ID mismatch: existing_device_id does not match validated device", status=400) + # Require force flag when device type mismatches, but only for actions that use it _FORCE_REQUIRED_ACTIONS = {"link", "update", "update_serial", "update_type"} force = request.POST.get("force") == "on" @@ -900,7 +907,10 @@ def post(self, request, device_id): except (TypeError, ValueError): return HttpResponse("Invalid or missing LibreNMS device_id in payload", status=400) - # Check for LibreNMS ID collision before any linking action + # Check for LibreNMS ID collision before any linking action. + # find_by_librenms_id returns None or the *one* device matching the ID. + # Comparing .pk != existing_device.pk is equivalent to .exclude(pk=...).exists() + # for conflict detection — both find any *other* device with the same librenms_id. if action in {"link", "update", "update_serial"}: from netbox_librenms_plugin.utils import find_by_librenms_id @@ -915,15 +925,11 @@ def post(self, request, device_id): if action == "link": # Link to LibreNMS and update name from LibreNMS data resolved_name = validation.get("resolved_name") - hostname = ( - resolved_name - if resolved_name - else _determine_device_name( - libre_device, - use_sysname=request.POST.get("use-sysname-toggle") == "on", - strip_domain=request.POST.get("strip-domain-toggle") == "on", - ) - ) + if resolved_name: + hostname = resolved_name + else: + use_sysname, strip_domain = _resolve_naming_preferences(request) + hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) existing_device.name = hostname if librenms_device_type: @@ -935,21 +941,17 @@ def post(self, request, device_id): # Update hostname, serial, and link to LibreNMS resolved_name = validation.get("resolved_name") incoming_serial = libre_device.get("serial") or "" - hostname = ( - resolved_name - if resolved_name - else _determine_device_name( - libre_device, - use_sysname=request.POST.get("use-sysname-toggle") == "on", - strip_domain=request.POST.get("strip-domain-toggle") == "on", - ) - ) + if resolved_name: + hostname = resolved_name + else: + use_sysname, strip_domain = _resolve_naming_preferences(request) + hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) if incoming_serial and incoming_serial != "-": conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() if conflict_device: return HttpResponse( - f"Serial conflict: '{incoming_serial}' is already assigned to device " - f"'{conflict_device.name}' (ID: {conflict_device.pk})", + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", status=409, ) existing_device.serial = incoming_serial @@ -970,8 +972,8 @@ def post(self, request, device_id): conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() if conflict_device: return HttpResponse( - f"Serial conflict: '{incoming_serial}' is already assigned to device " - f"'{conflict_device.name}' (ID: {conflict_device.pk})", + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", status=409, ) existing_device.serial = incoming_serial @@ -987,15 +989,11 @@ def post(self, request, device_id): elif action == "sync_name": # Sync device name from LibreNMS (e.g., IP → sysName) resolved_name = validation.get("resolved_name") - hostname = ( - resolved_name - if resolved_name - else _determine_device_name( - libre_device, - use_sysname=request.POST.get("use-sysname-toggle") == "on", - strip_domain=request.POST.get("strip-domain-toggle") == "on", - ) - ) + if resolved_name: + hostname = resolved_name + else: + use_sysname, strip_domain = _resolve_naming_preferences(request) + hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) existing_device.name = hostname existing_device.save() logger.info(f"Synced name on device '{existing_device.name}' from LibreNMS") @@ -1021,8 +1019,8 @@ def post(self, request, device_id): f"'{conflict_device.name}' (pk={conflict_device.pk})" ) return HttpResponse( - f"Serial conflict: '{incoming_serial}' is already assigned to device " - f"'{conflict_device.name}' (ID: {conflict_device.pk})", + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", status=409, ) existing_device.serial = incoming_serial From 61dab4aa95a5197ee3ca223650e6f75a1c120776 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 10:15:33 +0100 Subject: [PATCH 23/39] fix: inventory-specific batch 4 fixes - bulk_import.py: run job cancellation check on first device too (idx==0) so a pre-stopped job is detected before processing any devices - bulk_import.py: use VirtualMachine model in _refresh_existing_device when import_as_vm is True, so newly-imported VM rows are found on re-check; skip device_role enrichment for VMs --- .../import_utils/bulk_import.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 62046941ae..9cc586d353 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -90,8 +90,8 @@ def bulk_import_devices_shared( api = LibreNMSAPI(server_key=server_key) for idx, device_id in enumerate(device_ids, start=1): - # Check for job cancellation every 5 devices - if job and idx % 5 == 0: + # Check for job cancellation on the first device and every 5 devices thereafter + if job and (idx == 0 or idx % 5 == 0): # Refresh job from DB to get current status job.job.refresh_from_db() job_status = job.job.status @@ -317,6 +317,10 @@ def _refresh_existing_device(validation: dict, libre_device: dict = None, server return try: from dcim.models import Device + from virtualization.models import VirtualMachine + + import_as_vm = validation.get("import_as_vm", False) + Model = VirtualMachine if import_as_vm else Device librenms_id = libre_device.get("device_id") hostname = libre_device.get("hostname", "") @@ -328,7 +332,7 @@ def _refresh_existing_device(validation: dict, libre_device: dict = None, server # Check by librenms_id custom field first (JSON multi-server format + legacy) if librenms_id: try: - new_device = find_by_librenms_id(Device, int(librenms_id), server_key) + new_device = find_by_librenms_id(Model, int(librenms_id), server_key) if new_device: match_type = "librenms_id" except (ValueError, TypeError): @@ -336,9 +340,9 @@ def _refresh_existing_device(validation: dict, libre_device: dict = None, server # Fall back to hostname match if not new_device and hostname: - new_device = Device.objects.filter(name__iexact=hostname).first() + new_device = Model.objects.filter(name__iexact=hostname).first() if not new_device and sys_name: - new_device = Device.objects.filter(name__iexact=sys_name).first() + new_device = Model.objects.filter(name__iexact=sys_name).first() if new_device: match_type = "hostname" @@ -347,7 +351,7 @@ def _refresh_existing_device(validation: dict, libre_device: dict = None, server validation["existing_match_type"] = match_type validation["can_import"] = False validation["is_ready"] = False - if hasattr(new_device, "role") and new_device.role: + if not import_as_vm and hasattr(new_device, "role") and new_device.role: validation["device_role"] = {"found": True, "role": new_device.role} except Exception as e: logger.error(f"Failed to check for newly imported device: {e}") From 17d58c0d4820f9375ca7a5d4b166f14f553e672b Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 11:24:02 +0100 Subject: [PATCH 24/39] fix: batch 5 PR review fixes - bulk_import.py: add _empty_return() helper, replace 5 inline '([], False) if return_cache_status else []' with calls to it - cache.py: fix docstring example (spurious '-' before hash digest) - filters.py: normalize status filter value to int before comparison so string '1'/'0' from form fields is handled correctly - virtual_chassis.py: preserve discovered SNMP vc_position when available; fall back to sequential counter only when position is absent - actions.py: require validated_existing to be non-None for conflict actions (missing target now returns 400 instead of proceeding) - actions.py: replace find_by_librenms_id collision check with an exclusion-aware queryset (.exclude(pk=).exists()) for cleaner intent - actions.py: verify legacy cf_value matches librenms_id before migration to prevent migrating stale/incorrect associations - test_import_utils.py: add existing_device to all conflict-action test validation dicts to match the new required-target enforcement - test_import_utils.py: patch find_matching_platform at utility module level instead of dcim.models.Platform --- .../import_utils/bulk_import.py | 17 +++++--- netbox_librenms_plugin/import_utils/cache.py | 2 +- .../import_utils/filters.py | 9 +++- .../import_utils/virtual_chassis.py | 15 +++++-- .../tests/test_import_utils.py | 31 ++++++++------ .../views/imports/actions.py | 42 +++++++++++++------ 6 files changed, 80 insertions(+), 36 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 9cc586d353..7fb65ea3e7 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -21,6 +21,11 @@ logger = logging.getLogger(__name__) +def _empty_return(return_cache_status: bool): + """Centralised empty-result return value for process_device_filters.""" + return ([], False) if return_cache_status else [] + + def bulk_import_devices_shared( device_ids: List[int], server_key: str = None, @@ -432,7 +437,7 @@ def process_device_filters( except (BrokenPipeError, ConnectionError, IOError) as e: if request: logger.info(f"Client disconnected during VC prefetch: {e}") - return ([], False) if return_cache_status else [] + return _empty_return(return_cache_status) raise # Validate each device @@ -452,13 +457,13 @@ def process_device_filters( if rq_job.is_failed or rq_job.is_stopped: job.logger.warning("Job was already stopped before validation started") - return ([], False) if return_cache_status else [] + return _empty_return(return_cache_status) except Exception: # Fall back to DB check if RQ check fails job.job.refresh_from_db() if job.job.status == JobStatusChoices.STATUS_FAILED: job.logger.warning("Job was stopped before validation started") - return ([], False) if return_cache_status else [] + return _empty_return(return_cache_status) else: logger.info(f"Validating {total} devices") @@ -481,13 +486,13 @@ def process_device_filters( job.logger.info( f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." ) - return ([], False) if return_cache_status else [] + return _empty_return(return_cache_status) except Exception: # If we can't check RQ status, fall back to DB status check job.job.refresh_from_db() if job.job.status == JobStatusChoices.STATUS_FAILED: job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") - return ([], False) if return_cache_status else [] + return _empty_return(return_cache_status) # Drop any cached validation/meta keys before recomputing device.pop("_validation", None) @@ -534,7 +539,7 @@ def process_device_filters( except (BrokenPipeError, ConnectionError, IOError) as e: if request: logger.info(f"Client disconnected during device validation: {e}") - return ([], False) if return_cache_status else [] + return _empty_return(return_cache_status) raise # Set VC detection metadata diff --git a/netbox_librenms_plugin/import_utils/cache.py b/netbox_librenms_plugin/import_utils/cache.py index 03f711ad02..627080b506 100644 --- a/netbox_librenms_plugin/import_utils/cache.py +++ b/netbox_librenms_plugin/import_utils/cache.py @@ -140,7 +140,7 @@ def get_validated_device_cache_key(server_key: str, filters: dict, device_id: in Example: >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) >>> key - 'validated_device_default_-e3b0c44298fc1c14_123_vc' + 'validated_device_default_e3b0c44298fc1c14_123_vc' """ # Sort filters for a deterministic, cross-process stable hash filter_hash = hashlib.sha256(json.dumps(sorted(filters.items()), sort_keys=True).encode()).hexdigest()[:16] diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py index 4085dfa325..f312f299c2 100644 --- a/netbox_librenms_plugin/import_utils/filters.py +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -87,10 +87,15 @@ def get_librenms_devices_for_import( if filters: # Check for status filter first - it has special handling if filters.get("status") is not None: + # Normalize to int: form fields send strings ("1"/"0"), API may send ints + try: + status_val = int(filters["status"]) + except (ValueError, TypeError): + status_val = None # Status filter uses special types that don't need query param - if filters["status"] == 1: + if status_val == 1: api_filters["type"] = "up" - elif filters["status"] == 0: + elif status_val == 0: api_filters["type"] = "down" # Save ALL other filters for client-side filtering when status is used diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py index 7f775faa79..70778098cc 100644 --- a/netbox_librenms_plugin/import_utils/virtual_chassis.py +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -402,7 +402,17 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") continue - member_name = _generate_vc_member_name(master_base_name, position, serial=serial, pattern=vc_pattern) + # Prefer the discovered SNMP position; fall back to sequential counter + try: + discovered_pos = int(member.get("position")) if member.get("position") is not None else None + except (TypeError, ValueError): + discovered_pos = None + chosen_pos = discovered_pos if discovered_pos is not None else position + # Advance sequential counter only when it was consumed as a fallback + if discovered_pos is None: + position += 1 + + member_name = _generate_vc_member_name(master_base_name, chosen_pos, serial=serial, pattern=vc_pattern) # Check for duplicate name if Device.objects.filter(name=member_name).exists(): @@ -419,12 +429,11 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis platform=master_device.platform, serial=serial, virtual_chassis=vc, - vc_position=position, + vc_position=chosen_pos, comments=f"VC member (LibreNMS: {member.get('name', 'Unknown')})\n" f"Auto-created from stack inventory", ) members_created += 1 - position += 1 # Validate member count expected_members = len( diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index ba88025fc3..b9d358762e 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1788,7 +1788,7 @@ def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache) "sysName": "switch-01.example.com", "serial": "ABC123", } - validation = {"can_import": False} + validation = {"can_import": False, "existing_device": existing_device} selections = {} request = self._create_request("link", 42, use_sysname=True) @@ -1801,6 +1801,7 @@ def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache) mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1829,7 +1830,7 @@ def test_update_action_sets_hostname_serial_and_librenms_id(self, mock_cache_key "sysName": "new-name.example.com", "serial": "NEW-SERIAL", } - validation = {"can_import": False} + validation = {"can_import": False, "existing_device": existing_device} selections = {} request = self._create_request("update", 42, use_sysname=True) @@ -1842,6 +1843,7 @@ def test_update_action_sets_hostname_serial_and_librenms_id(self, mock_cache_key mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1866,7 +1868,7 @@ def test_update_serial_action_updates_serial_only(self, mock_cache_key, mock_cac existing_device.serial = "OLD-SERIAL" libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "NEW-SERIAL"} - validation = {"can_import": False} + validation = {"can_import": False, "existing_device": existing_device} selections = {} request = self._create_request("update_serial", 42) @@ -1879,6 +1881,7 @@ def test_update_serial_action_updates_serial_only(self, mock_cache_key, mock_cac mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1904,7 +1907,7 @@ def test_update_skips_dash_serial(self, mock_cache_key, mock_cache): existing_device.serial = "EXISTING" libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "-"} - validation = {"can_import": False} + validation = {"can_import": False, "existing_device": existing_device} selections = {} request = self._create_request("update_serial", 42) @@ -1971,7 +1974,7 @@ def test_sync_name_action_updates_name(self, mock_cache_key, mock_cache): "sysName": "switch-01.example.com", "serial": "ABC123", } - validation = {"can_import": False} + validation = {"can_import": False, "existing_device": existing_device} selections = {} request = self._create_request("sync_name", 42, use_sysname=True) @@ -2002,7 +2005,7 @@ def test_device_type_mismatch_blocked_without_force(self, mock_cache_key, mock_c existing_device.custom_field_data = {} libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "ABC123"} - validation = {"can_import": False, "device_type_mismatch": True} + validation = {"can_import": False, "device_type_mismatch": True, "existing_device": existing_device} selections = {} request = self._create_request("link", 42, use_sysname=True) @@ -2037,7 +2040,7 @@ def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cach "sysName": "switch-01.example.com", "serial": "ABC123", } - validation = {"can_import": False, "device_type_mismatch": True} + validation = {"can_import": False, "device_type_mismatch": True, "existing_device": existing_device} selections = {} request = self._create_request("link", 42, use_sysname=True) @@ -2051,6 +2054,7 @@ def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cach mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -2083,6 +2087,7 @@ def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cach "can_import": False, "device_type_mismatch": True, "device_type": {"device_type": librenms_device_type}, + "existing_device": existing_device, } selections = {} @@ -2097,6 +2102,7 @@ def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cach mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -2131,6 +2137,7 @@ def test_update_type_action_changes_device_type(self, mock_cache_key, mock_cache "can_import": False, "device_type_mismatch": True, "device_type": {"device_type": new_device_type}, + "existing_device": existing_device, } selections = {} @@ -2174,6 +2181,7 @@ def test_sync_serial_action(self, mock_cache_key, mock_cache): mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -2202,13 +2210,12 @@ def test_sync_platform_action(self, mock_cache_key, mock_cache): patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, - # Patch at dcim.models level: find_matching_platform uses an inline - # 'from dcim.models import Platform' so patching dcim.models.Platform - # correctly intercepts the binding at call time. - patch("dcim.models.Platform") as mock_platform_cls, + # Patch find_matching_platform at the utility module level — the action imports + # it from netbox_librenms_plugin.utils, so that is the correct seam to mock. + patch("netbox_librenms_plugin.utils.find_matching_platform") as mock_find_platform, ): mock_device_cls.objects.get.return_value = existing_device - mock_platform_cls.objects.get.return_value = mock_platform + mock_find_platform.return_value = {"found": True, "platform": mock_platform, "match_type": "exact"} mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 41e19b669f..e98bbbe539 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -881,10 +881,12 @@ def post(self, request, device_id): return HttpResponse("LibreNMS device not found", status=404) # Verify the POSTed existing_device_id matches the validated conflict target. - # Without this check, an attacker could mutate an arbitrary device. - # Only enforce when validation has a known existing_device (conflict case). + # Require a confirmed conflict target: if validation has no existing_device, the + # LibreNMS device was not validated against this NetBox device, so mutations are unsafe. validated_existing = validation.get("existing_device") if validation else None - if validated_existing is not None and validated_existing.pk != existing_device.pk: + if validated_existing is None: + return HttpResponse("Missing validated conflict target", status=400) + if validated_existing.pk != existing_device.pk: return HttpResponse("Device ID mismatch: existing_device_id does not match validated device", status=400) # Require force flag when device type mismatches, but only for actions that use it @@ -907,18 +909,23 @@ def post(self, request, device_id): except (TypeError, ValueError): return HttpResponse("Invalid or missing LibreNMS device_id in payload", status=400) - # Check for LibreNMS ID collision before any linking action. - # find_by_librenms_id returns None or the *one* device matching the ID. - # Comparing .pk != existing_device.pk is equivalent to .exclude(pk=...).exists() - # for conflict detection — both find any *other* device with the same librenms_id. + # Check for LibreNMS ID collision before any linking action: detect any *other* + # Device that already carries the same librenms_id, excluding the current target. if action in {"link", "update", "update_serial"}: - from netbox_librenms_plugin.utils import find_by_librenms_id + from django.db.models import Q - id_conflict = find_by_librenms_id(Device, librenms_id, self.librenms_api.server_key) - if id_conflict and id_conflict.pk != existing_device.pk: + server_key = self.librenms_api.server_key + conflict_exists = ( + Device.objects.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}) + | Q(custom_field_data__librenms_id=librenms_id) + ) + .exclude(pk=existing_device.pk) + .exists() + ) + if conflict_exists: return HttpResponse( - f"LibreNMS ID conflict: ID {librenms_id} is already assigned to device " - f"'{escape(id_conflict.name)}' (ID: {id_conflict.pk})", + f"LibreNMS ID conflict: ID {librenms_id} is already assigned to another device.", status=409, ) @@ -1064,12 +1071,23 @@ def post(self, request, device_id): # confirmed by serial match (or explicit force). from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + # Direct access needed to detect legacy integer format for migration prompt: + # LibreNMSAPI.get_librenms_id() returns an int in both formats; only the raw + # type check on custom_field_data reveals whether migration is needed. cf_value = existing_device.custom_field_data.get("librenms_id") if not isinstance(cf_value, int): return HttpResponse( "Device librenms_id is already in JSON format; no migration needed.", status=400, ) + # Verify the stored legacy ID matches the active LibreNMS device_id so we don't + # migrate a stale/incorrect association to the wrong server mapping. + if cf_value != librenms_id: + return HttpResponse( + f"Legacy librenms_id ({cf_value}) does not match the active device ID " + f"({librenms_id}); cannot migrate safely.", + status=400, + ) if not validation.get("serial_confirmed") and not force: return HttpResponse( "Serial number not confirmed. Check the force checkbox to migrate without serial verification.", From d7ef8ac427bcf0cb3cb967e05211a06890737059 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 11:24:41 +0100 Subject: [PATCH 25/39] fix: inventory-only: fix idx==0 cancellation check (enumerate starts at 1) --- netbox_librenms_plugin/import_utils/bulk_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 7fb65ea3e7..2314dc2819 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -96,7 +96,7 @@ def bulk_import_devices_shared( for idx, device_id in enumerate(device_ids, start=1): # Check for job cancellation on the first device and every 5 devices thereafter - if job and (idx == 0 or idx % 5 == 0): + if job and (idx == 1 or idx % 5 == 0): # Refresh job from DB to get current status job.job.refresh_from_db() job_status = job.job.status From 0fa8e0715f6c5caab8fe1b46281515c5de3b0860 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 14:01:26 +0100 Subject: [PATCH 26/39] fix: batch 6 PR review fixes - bulk_import.py: normalize status (int/str) in show_disabled filter - bulk_import.py: use api.server_key (resolved) in import_single_device call - filters.py: normalize status in count_librenms_devices show_disabled filter - virtual_chassis.py: fix 0-based fallback for vc_position (idx+1 not idx) - virtual_chassis.py: advance sequential counter past discovered_pos; normalize pos=0 to absent - actions.py: add django.db.transaction import - actions.py: module-level _FORCE_REQUIRED_ACTIONS frozenset - actions.py: defensive .get('resolved_name') instead of [] access - actions.py: add NetBoxObjectPermissionMixin to DeviceConflictActionView - actions.py: wrap librenms_id collision check+write in transaction.atomic() - tests: fix unknown-action test to include existing_device in validation dict - tests: update warning text assertion to 'hostname differs' - tests: add transaction mock to 6 link/update/update_serial tests - tests: add TestProcessDeviceFilters with 5 new tests for status norm + api.server_key --- .../import_utils/bulk_import.py | 14 +- .../import_utils/filters.py | 12 +- .../import_utils/virtual_chassis.py | 17 +- .../tests/test_import_utils.py | 186 +++++++++++++++++- .../views/imports/actions.py | 175 ++++++++-------- 5 files changed, 315 insertions(+), 89 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 2314dc2819..159af22396 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -155,7 +155,7 @@ def bulk_import_devices_shared( result = import_single_device( device_id, - server_key=server_key, + server_key=api.server_key, # use resolved key, not raw parameter (may be None) sync_options=sync_options, manual_mappings=device_mappings if device_mappings else None, libre_device=libre_device, @@ -411,9 +411,17 @@ def process_device_filters( return_cache_status=True, ) - # Filter out disabled devices if requested + # Filter out disabled devices if requested; normalize status to int to handle + # both integer (1) and string ("1") responses from the LibreNMS API. if not show_disabled: - libre_devices = [d for d in libre_devices if d.get("status") == 1] + + def _is_active(d): + try: + return int(d.get("status", 0)) == 1 + except (TypeError, ValueError): + return False + + libre_devices = [d for d in libre_devices if _is_active(d)] if job: job.logger.info(f"Found {len(libre_devices)} devices to process") diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py index f312f299c2..a7890c9293 100644 --- a/netbox_librenms_plugin/import_utils/filters.py +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -35,9 +35,17 @@ def get_device_count_for_filters( """ devices = get_librenms_devices_for_import(api, filters=filters, force_refresh=clear_cache) - # Filter out disabled devices if requested + # Filter out disabled devices if requested; normalize status to int to handle + # both integer (1) and string ("1") responses from the LibreNMS API. if not show_disabled: - devices = [d for d in devices if d.get("status") == 1] + + def _is_active(d): + try: + return int(d.get("status", 0)) == 1 + except (TypeError, ValueError): + return False + + devices = [d for d in devices if _is_active(d)] return len(devices) diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py index 70778098cc..2f030875e4 100644 --- a/netbox_librenms_plugin/import_utils/virtual_chassis.py +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -198,11 +198,13 @@ def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> d # Step 5: Extract member info members = [] for idx, chassis in enumerate(chassis_items): - raw_position = chassis.get("entPhysicalParentRelPos", idx) + # entPhysicalParentRelPos is 1-based; fall back to idx+1 (not idx) so + # position 0 is never produced — VC positions must be ≥ 1. + raw_position = chassis.get("entPhysicalParentRelPos", idx + 1) try: position = int(raw_position) except (TypeError, ValueError): - position = idx + position = idx + 1 member_data = { "serial": chassis.get("entPhysicalSerialNum", ""), "position": position, @@ -402,15 +404,22 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") continue - # Prefer the discovered SNMP position; fall back to sequential counter + # Prefer the discovered SNMP position; fall back to sequential counter. + # Normalize discovered_pos: 0 is not a valid VC position, treat as absent. try: discovered_pos = int(member.get("position")) if member.get("position") is not None else None except (TypeError, ValueError): discovered_pos = None + if discovered_pos is not None and discovered_pos < 1: + discovered_pos = None # 0 is invalid for vc_position; fall back to counter chosen_pos = discovered_pos if discovered_pos is not None else position - # Advance sequential counter only when it was consumed as a fallback + # Advance the sequential counter: + # - if discovered_pos was used, advance counter past it to avoid future reuse; + # - if counter was consumed as fallback, increment it normally. if discovered_pos is None: position += 1 + else: + position = max(position, discovered_pos + 1) member_name = _generate_vc_member_name(master_base_name, chosen_pos, serial=serial, pattern=vc_pattern) diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index b9d358762e..39ea7e72a1 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1215,7 +1215,7 @@ def device_filter(**kwargs): assert result["serial_action"] == "hostname_differs" assert result["existing_match_type"] == "serial" - assert "reinstalled" in result["warnings"][0] + assert "hostname differs" in result["warnings"][0] def test_hostname_match_diff_serial_offers_update(self): """Hostname matches but serial differs offers update_serial action.""" @@ -1797,7 +1797,9 @@ def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache) patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None @@ -1839,7 +1841,9 @@ def test_update_action_sets_hostname_serial_and_librenms_id(self, mock_cache_key patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None @@ -1877,7 +1881,9 @@ def test_update_serial_action_updates_serial_only(self, mock_cache_key, mock_cac patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None @@ -1916,8 +1922,11 @@ def test_update_skips_dash_serial(self, mock_cache_key, mock_cache): patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1943,14 +1952,18 @@ def test_unknown_action_returns_400(self): request = self._create_request("invalid_action", 42) existing_device = MagicMock() + existing_device.pk = 42 libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "ABC"} with ( patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "require_object_permissions", return_value=None), patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device - mock_validate.return_value = (libre_device, {}, {}) + # Include existing_device so the validated-conflict-target guard passes; + # we want to exercise the unknown-action branch, not the missing-device guard. + mock_validate.return_value = (libre_device, {"existing_device": existing_device}, {}) response = view.post(request, device_id=10) @@ -2050,7 +2063,9 @@ def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cach patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None @@ -2098,7 +2113,9 @@ def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cach patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None @@ -2529,3 +2546,168 @@ def test_backward_compatible_defaults(self, *mocks): result = validate_device_for_import(device_data, include_vc_detection=False) assert "resolved_name" in result assert result["resolved_name"] == "switch-01" + + +class TestProcessDeviceFilters: + """Tests for process_device_filters and related bulk_import utilities.""" + + def test_show_disabled_filters_integer_status_1(self): + """show_disabled=False should keep devices with status==1 (int).""" + from unittest.mock import MagicMock, patch + + devices = [ + {"device_id": 1, "hostname": "a", "status": 1}, + {"device_id": 2, "hostname": "b", "status": 0}, + ] + with ( + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_librenms_devices_for_import", + return_value=(devices, False), + ), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import", + side_effect=lambda d, **kw: { + "resolved_name": d["hostname"], + "is_ready": True, + "can_import": True, + "status": "active", + "existing_device": None, + "import_as_vm": False, + "existing_match_type": None, + }, + ), + patch("netbox_librenms_plugin.import_utils.bulk_import.prefetch_vc_data_for_devices"), + patch("netbox_librenms_plugin.import_utils.bulk_import.cache"), + patch("netbox_librenms_plugin.import_utils.bulk_import.get_cache_metadata_key", return_value="key"), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_validated_device_cache_key", return_value="vkey" + ), + ): + from netbox_librenms_plugin.import_utils.bulk_import import process_device_filters + + api = MagicMock() + api.server_key = "default" + result = process_device_filters( + api, filters={}, vc_detection_enabled=False, clear_cache=False, show_disabled=False + ) + + # Only device with status==1 should be processed; disabled device excluded before validation + assert len(result) == 1 + assert result[0]["hostname"] == "a" + + def test_show_disabled_filters_string_status_1(self): + """show_disabled=False should keep devices with status=='1' (string from API).""" + from unittest.mock import MagicMock, patch + + devices = [ + {"device_id": 1, "hostname": "a", "status": "1"}, + {"device_id": 2, "hostname": "b", "status": "0"}, + ] + with ( + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_librenms_devices_for_import", + return_value=(devices, False), + ), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import", + side_effect=lambda d, **kw: { + "resolved_name": d["hostname"], + "is_ready": True, + "can_import": True, + "status": "active", + "existing_device": None, + "import_as_vm": False, + "existing_match_type": None, + }, + ), + patch("netbox_librenms_plugin.import_utils.bulk_import.prefetch_vc_data_for_devices"), + patch("netbox_librenms_plugin.import_utils.bulk_import.cache"), + patch("netbox_librenms_plugin.import_utils.bulk_import.get_cache_metadata_key", return_value="key"), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_validated_device_cache_key", return_value="vkey" + ), + ): + from netbox_librenms_plugin.import_utils.bulk_import import process_device_filters + + api = MagicMock() + api.server_key = "default" + result = process_device_filters( + api, filters={}, vc_detection_enabled=False, clear_cache=False, show_disabled=False + ) + + assert len(result) == 1 + assert result[0]["hostname"] == "a" + + def test_show_disabled_true_includes_all(self): + """show_disabled=True should include both active and inactive devices.""" + from unittest.mock import MagicMock, patch + + devices = [ + {"device_id": 1, "hostname": "a", "status": 1}, + {"device_id": 2, "hostname": "b", "status": 0}, + ] + with ( + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_librenms_devices_for_import", + return_value=(devices, False), + ), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import", + side_effect=lambda d, **kw: { + "resolved_name": d["hostname"], + "is_ready": True, + "can_import": True, + "status": "active", + "existing_device": None, + "import_as_vm": False, + "existing_match_type": None, + }, + ), + patch("netbox_librenms_plugin.import_utils.bulk_import.prefetch_vc_data_for_devices"), + patch("netbox_librenms_plugin.import_utils.bulk_import.cache"), + patch("netbox_librenms_plugin.import_utils.bulk_import.get_cache_metadata_key", return_value="key"), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_validated_device_cache_key", return_value="vkey" + ), + ): + from netbox_librenms_plugin.import_utils.bulk_import import process_device_filters + + api = MagicMock() + api.server_key = "default" + result = process_device_filters( + api, filters={}, vc_detection_enabled=False, clear_cache=False, show_disabled=True + ) + + assert len(result) == 2 + + def test_empty_return_helper(self): + """_empty_return should return ([], False) when return_cache_status=True, else [].""" + from netbox_librenms_plugin.import_utils.bulk_import import _empty_return + + assert _empty_return(True) == ([], False) + assert _empty_return(False) == [] + + def test_bulk_import_devices_uses_resolved_server_key(self): + """bulk_import_devices_shared should pass api.server_key to import_single_device.""" + from unittest.mock import MagicMock, patch + + with ( + patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") as mock_api_cls, + patch("netbox_librenms_plugin.import_utils.bulk_import.import_single_device") as mock_import, + patch("netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import"), + patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions"), + ): + mock_api = MagicMock() + mock_api.server_key = "resolved-key" + mock_api.get_device_info.return_value = (True, {"device_id": 1, "hostname": "sw"}) + mock_api_cls.return_value = mock_api + mock_import.return_value = {"success": True, "device": MagicMock(), "is_vm": False} + + user = MagicMock() + from netbox_librenms_plugin.import_utils.bulk_import import bulk_import_devices_shared + + bulk_import_devices_shared([1], user=user, server_key=None) + + # The resolved api.server_key ("resolved-key") must be passed, not None + assert mock_import.call_args is not None + assert mock_import.call_args.kwargs.get("server_key") == "resolved-key" diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index e98bbbe539..ac724088d3 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -6,6 +6,7 @@ from django.contrib import messages from django.core.cache import cache from django.core.exceptions import PermissionDenied +from django.db import transaction from django.http import HttpResponse, JsonResponse from django.shortcuts import redirect, render from django.utils.html import escape @@ -31,10 +32,13 @@ ) from netbox_librenms_plugin.tables.device_status import DeviceImportTable from netbox_librenms_plugin.utils import get_user_pref, save_user_pref, set_librenms_device_id -from netbox_librenms_plugin.views.mixins import LibreNMSAPIMixin, LibreNMSPermissionMixin +from netbox_librenms_plugin.views.mixins import LibreNMSAPIMixin, LibreNMSPermissionMixin, NetBoxObjectPermissionMixin logger = logging.getLogger(__name__) +# Actions that require the force checkbox when a device-type mismatch is detected. +_FORCE_REQUIRED_ACTIONS = frozenset({"link", "update", "update_serial", "update_type"}) + def _resolve_naming_preferences(request) -> tuple[bool, bool]: """Resolve use_sysname/strip_domain: POST data → user pref → plugin settings.""" @@ -309,7 +313,7 @@ def post(self, request): vc_requested = request.GET.get("enable_vc_detection") == "true" validation["_vc_detection_enabled"] = vc_requested - device_name = validation["resolved_name"] + device_name = validation.get("resolved_name") if validation.get("virtual_chassis", {}).get("is_stack") and device_name: validation["virtual_chassis"] = update_vc_member_suggested_names( @@ -855,7 +859,9 @@ def post(self, request, device_id): return self.render_device_row(request, libre_device, validation, selections) -class DeviceConflictActionView(LibreNMSPermissionMixin, LibreNMSAPIMixin, DeviceImportHelperMixin, View): +class DeviceConflictActionView( + LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, LibreNMSAPIMixin, DeviceImportHelperMixin, View +): """HTMX view to resolve device conflicts (link, update, update serial).""" def post(self, request, device_id): @@ -876,6 +882,11 @@ def post(self, request, device_id): except (Device.DoesNotExist, ValueError): return HttpResponse("Existing device not found", status=404) + # Object-level change permission for the specific device being mutated. + self.required_object_permissions = {"POST": [("change", Device)]} + if error := self.require_object_permissions("POST"): + return error + libre_device, validation, selections = self.get_validated_device_with_selections(device_id, request) if not libre_device: return HttpResponse("LibreNMS device not found", status=404) @@ -890,7 +901,6 @@ def post(self, request, device_id): return HttpResponse("Device ID mismatch: existing_device_id does not match validated device", status=400) # Require force flag when device type mismatches, but only for actions that use it - _FORCE_REQUIRED_ACTIONS = {"link", "update", "update_serial", "update_type"} force = request.POST.get("force") == "on" if validation.get("device_type_mismatch") and action in _FORCE_REQUIRED_ACTIONS and not force: return HttpResponse( @@ -909,89 +919,98 @@ def post(self, request, device_id): except (TypeError, ValueError): return HttpResponse("Invalid or missing LibreNMS device_id in payload", status=400) - # Check for LibreNMS ID collision before any linking action: detect any *other* - # Device that already carries the same librenms_id, excluding the current target. + # Wrap the LibreNMS-ID collision check and subsequent write in a single + # transaction so the read-then-write is atomic for link/update/update_serial. if action in {"link", "update", "update_serial"}: from django.db.models import Q - server_key = self.librenms_api.server_key - conflict_exists = ( - Device.objects.filter( - Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}) - | Q(custom_field_data__librenms_id=librenms_id) - ) - .exclude(pk=existing_device.pk) - .exists() - ) - if conflict_exists: - return HttpResponse( - f"LibreNMS ID conflict: ID {librenms_id} is already assigned to another device.", - status=409, + with transaction.atomic(): + server_key = self.librenms_api.server_key + conflict_exists = ( + Device.objects.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}) + | Q(custom_field_data__librenms_id=librenms_id) + ) + .exclude(pk=existing_device.pk) + .exists() ) - - if action == "link": - # Link to LibreNMS and update name from LibreNMS data - resolved_name = validation.get("resolved_name") - if resolved_name: - hostname = resolved_name - else: - use_sysname, strip_domain = _resolve_naming_preferences(request) - hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) - set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) - existing_device.name = hostname - if librenms_device_type: - existing_device.device_type = librenms_device_type - existing_device.save() - logger.info(f"Linked device '{existing_device.name}' to LibreNMS ID {librenms_id}") - - elif action == "update": - # Update hostname, serial, and link to LibreNMS - resolved_name = validation.get("resolved_name") - incoming_serial = libre_device.get("serial") or "" - if resolved_name: - hostname = resolved_name - else: - use_sysname, strip_domain = _resolve_naming_preferences(request) - hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) - if incoming_serial and incoming_serial != "-": - conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() - if conflict_device: + if conflict_exists: return HttpResponse( - f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " - f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", + f"LibreNMS ID conflict: ID {librenms_id} is already assigned to another device.", status=409, ) - existing_device.serial = incoming_serial - existing_device.name = hostname - if librenms_device_type: - existing_device.device_type = librenms_device_type - set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) - existing_device.save() - logger.info( - f"Updated device '{existing_device.name}': serial={incoming_serial}, " - f"linked to LibreNMS ID {librenms_id}" - ) - elif action == "update_serial": - # Update only the serial and link to LibreNMS - incoming_serial = libre_device.get("serial") or "" - if incoming_serial and incoming_serial != "-": - conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() - if conflict_device: - return HttpResponse( - f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " - f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", - status=409, + if action == "link": + # Link to LibreNMS and update name from LibreNMS data + resolved_name = validation.get("resolved_name") + if resolved_name: + hostname = resolved_name + else: + use_sysname, strip_domain = _resolve_naming_preferences(request) + hostname = _determine_device_name( + libre_device, use_sysname=use_sysname, strip_domain=strip_domain + ) + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) + existing_device.name = hostname + if librenms_device_type: + existing_device.device_type = librenms_device_type + existing_device.save() + logger.info(f"Linked device '{existing_device.name}' to LibreNMS ID {librenms_id}") + + elif action == "update": + # Update hostname, serial, and link to LibreNMS + resolved_name = validation.get("resolved_name") + incoming_serial = libre_device.get("serial") or "" + if resolved_name: + hostname = resolved_name + else: + use_sysname, strip_domain = _resolve_naming_preferences(request) + hostname = _determine_device_name( + libre_device, use_sysname=use_sysname, strip_domain=strip_domain + ) + if incoming_serial and incoming_serial != "-": + conflict_device = ( + Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + ) + if conflict_device: + return HttpResponse( + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", + status=409, + ) + existing_device.serial = incoming_serial + existing_device.name = hostname + if librenms_device_type: + existing_device.device_type = librenms_device_type + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) + existing_device.save() + logger.info( + f"Updated device '{existing_device.name}': serial={incoming_serial}, " + f"linked to LibreNMS ID {librenms_id}" + ) + + elif action == "update_serial": + # Update only the serial and link to LibreNMS + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-": + conflict_device = ( + Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + ) + if conflict_device: + return HttpResponse( + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", + status=409, + ) + existing_device.serial = incoming_serial + if librenms_device_type: + existing_device.device_type = librenms_device_type + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) + existing_device.save() + logger.info( + f"Updated serial on device '{existing_device.name}' to {incoming_serial}, " + f"linked to LibreNMS ID {librenms_id}" ) - existing_device.serial = incoming_serial - if librenms_device_type: - existing_device.device_type = librenms_device_type - set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) - existing_device.save() - logger.info( - f"Updated serial on device '{existing_device.name}' to {incoming_serial}, " - f"linked to LibreNMS ID {librenms_id}" - ) elif action == "sync_name": # Sync device name from LibreNMS (e.g., IP → sysName) From 4506ba6876669313890ab8a827faaa1ec4ea9d06 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 17:11:54 +0100 Subject: [PATCH 27/39] fix: batch 7 - RQ-first cancellation, VC position off-by-one, platform_synced bool - bulk_import_devices_shared: check RQ/Redis job state first (is_stopped/is_failed) before falling back to DB status, matching pattern in process_device_filters(); also check on idx==1 (first iteration) not just idx%5==0 - virtual_chassis._clone_virtual_chassis_data: position fallback now idx+1 (1-based) instead of idx (0-based) so position 0 is never produced - virtual_chassis.detect_virtual_chassis_from_inventory: suggested_name uses position directly (not position+1) since position is already 1-based after batch 6 fallback fix - actions._build_sync_info: wrap platform_synced expression in bool() so lazy and/or chain never returns None when matching_platform is None or netbox_platform is None; all sync flag types are now stable booleans - tests: add TestVCPositionHandling (4 tests) and TestBulkImportCancellation (4 tests); add 2 TestBuildSyncInfo tests for platform_synced=False type-stability --- .../import_utils/bulk_import.py | 42 ++-- .../import_utils/virtual_chassis.py | 11 +- .../tests/test_import_utils.py | 189 ++++++++++++++++++ .../views/imports/actions.py | 2 +- 4 files changed, 225 insertions(+), 19 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 159af22396..f943337ab8 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -95,20 +95,36 @@ def bulk_import_devices_shared( api = LibreNMSAPI(server_key=server_key) for idx, device_id in enumerate(device_ids, start=1): - # Check for job cancellation on the first device and every 5 devices thereafter + # Check for job cancellation on first iteration and every 5th thereafter. + # Check RQ/Redis state first (reflects stop API immediately); fall back to DB. if job and (idx == 1 or idx % 5 == 0): - # Refresh job from DB to get current status - job.job.refresh_from_db() - job_status = job.job.status - status_value = job_status.value if hasattr(job_status, "value") else job_status - if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): - if job.logger: - job.logger.warning(f"Import job cancelled at device {idx} of {total}") - else: - logger.warning(f"Import cancelled at device {idx} of {total}") - break - # Log progress - if job.logger: + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + if rq_job.is_failed or rq_job.is_stopped: + if job.logger: + job.logger.warning( + f"Import job stopped at device {idx} of {total} (RQ status: {rq_job.get_status()})" + ) + else: + logger.warning(f"Import cancelled at device {idx} of {total}") + break + except Exception: + # Fall back to DB check if RQ is unavailable + job.job.refresh_from_db() + job_status = job.job.status + status_value = job_status.value if hasattr(job_status, "value") else job_status + if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): + if job.logger: + job.logger.warning(f"Import job cancelled at device {idx} of {total}") + else: + logger.warning(f"Import cancelled at device {idx} of {total}") + break + # Log progress (only after first check) + if idx > 1 and job.logger: job.logger.info(f"Imported device {idx} of {total}") try: diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py index 2f030875e4..a3c474d393 100644 --- a/netbox_librenms_plugin/import_utils/virtual_chassis.py +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -32,11 +32,11 @@ def _clone_virtual_chassis_data(data: dict | None) -> dict: members = [] for idx, member in enumerate(data.get("members", [])): member_copy = member.copy() - raw_position = member_copy.get("position", idx) + raw_position = member_copy.get("position", idx + 1) try: member_copy["position"] = int(raw_position) except (TypeError, ValueError): - member_copy["position"] = idx + member_copy["position"] = idx + 1 # 1-based fallback; position 0 is invalid members.append(member_copy) member_count = data.get("member_count") or len(members) @@ -214,11 +214,12 @@ def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> d "description": chassis.get("entPhysicalDescr", ""), } - # Generate suggested name if we have master name + # Generate suggested name if we have master name. + # position is already 1-based, so pass it directly (no +1). if master_name: - member_data["suggested_name"] = _generate_vc_member_name(master_name, position + 1) + member_data["suggested_name"] = _generate_vc_member_name(master_name, position) else: - member_data["suggested_name"] = f"Member-{position + 1}" + member_data["suggested_name"] = f"Member-{position}" members.append(member_data) diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 39ea7e72a1..932af5a346 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -2349,6 +2349,50 @@ def test_platform_out_of_sync(self): assert result["platform_synced"] is False assert result["all_synced"] is False + def test_platform_no_match_found_returns_bool(self): + """When find_matching_platform returns no match, platform_synced must be False (not None).""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "ABC123" + existing.platform = MagicMock() # device has a platform set + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + libre_device = {"serial": "ABC123", "os": "ios", "hardware": "-"} + + with patch("netbox_librenms_plugin.utils.find_matching_platform") as mock_platform_match: + mock_platform_match.return_value = {"found": False, "platform": None} + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + # Without bool() cast this would be None; verify it's exactly False (type-stable) + assert result["platform_synced"] is False + assert isinstance(result["platform_synced"], bool) + + def test_platform_synced_no_netbox_platform_returns_bool(self): + """When device has no platform in NetBox and os is non-dash, platform_synced must be bool.""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "ABC123" + existing.platform = None # no platform on device + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + libre_device = {"serial": "ABC123", "os": "eos", "hardware": "-"} + + with patch("netbox_librenms_plugin.utils.find_matching_platform") as mock_platform_match: + mock_platform_match.return_value = {"found": True, "platform": MagicMock()} + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + # None and ... returns None; bool() cast ensures False + assert result["platform_synced"] is False + assert isinstance(result["platform_synced"], bool) + def test_hardware_no_match_device_type_out_of_sync(self): """When hardware is present but no device type match found, device_type_synced is False.""" from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView @@ -2711,3 +2755,148 @@ def test_bulk_import_devices_uses_resolved_server_key(self): # The resolved api.server_key ("resolved-key") must be passed, not None assert mock_import.call_args is not None assert mock_import.call_args.kwargs.get("server_key") == "resolved-key" + + +class TestVCPositionHandling: + """Test VC position normalization and suggested name generation.""" + + def test_clone_vc_data_position_fallback_is_one_based(self): + """_clone_virtual_chassis_data fallback must be 1-based (idx+1, not idx).""" + from netbox_librenms_plugin.import_utils.virtual_chassis import _clone_virtual_chassis_data + + data = {"is_stack": True, "member_count": 2, "members": [{"serial": "S1"}, {"serial": "S2"}]} + result = _clone_virtual_chassis_data(data) + positions = [m["position"] for m in result["members"]] + # First member: idx=0 → position should be 1, not 0 + assert positions[0] == 1 + assert positions[1] == 2 + + def test_clone_vc_data_preserves_explicit_positions(self): + """_clone_virtual_chassis_data must preserve explicitly set positions.""" + from netbox_librenms_plugin.import_utils.virtual_chassis import _clone_virtual_chassis_data + + data = { + "is_stack": True, + "member_count": 2, + "members": [{"serial": "S1", "position": 3}, {"serial": "S2", "position": 5}], + } + result = _clone_virtual_chassis_data(data) + assert result["members"][0]["position"] == 3 + assert result["members"][1]["position"] == 5 + + def test_clone_vc_data_bad_position_falls_back_to_one_based(self): + """_clone_virtual_chassis_data falls back to idx+1 for non-int position.""" + from netbox_librenms_plugin.import_utils.virtual_chassis import _clone_virtual_chassis_data + + data = { + "is_stack": True, + "member_count": 2, + "members": [{"serial": "S1", "position": "bad"}, {"serial": "S2", "position": None}], + } + result = _clone_virtual_chassis_data(data) + # idx=0 → fallback 1, idx=1 → fallback 2 + assert result["members"][0]["position"] == 1 + assert result["members"][1]["position"] == 2 + + def test_suggested_name_uses_position_directly(self): + """Suggested name generation must use position directly (not position+1). + + This test verifies that _generate_vc_member_name is called with the + already-1-based position value, not position+1. + """ + from netbox_librenms_plugin.import_utils.virtual_chassis import _generate_vc_member_name + + # position=1 should produce name with "1", not "2" + name = _generate_vc_member_name("switch-1", 1, pattern="-M{position}") + assert name == "switch-1-M1", f"Expected 'switch-1-M1', got '{name}'" + + # position=2 should produce "2", not "3" + name = _generate_vc_member_name("switch-1", 2, pattern="-M{position}") + assert name == "switch-1-M2", f"Expected 'switch-1-M2', got '{name}'" + + +class TestBulkImportCancellation: + """Test that bulk_import_devices_shared respects RQ and DB cancellation.""" + + def _run_bulk_import(self, mock_rq_job=None, db_status="running", device_ids=None): + """Helper: run bulk_import with provided mocks, return import call count.""" + from unittest.mock import MagicMock, patch + + if device_ids is None: + device_ids = [1, 2, 3, 4, 5, 6] + + job = MagicMock() + job.job.job_id = "test-uuid" + job_status = MagicMock() + job_status.value = db_status + job.job.status = job_status + job.logger = MagicMock() + + with ( + patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") as mock_api_cls, + patch("netbox_librenms_plugin.import_utils.bulk_import.import_single_device") as mock_import, + patch("netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import"), + patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions"), + # Inline imports in the loop use django_rq.get_queue / rq.job.Job directly + patch("django_rq.get_queue") as mock_get_queue, + patch("rq.job.Job") as mock_rqjob_cls, + ): + mock_api = MagicMock() + mock_api.server_key = "default" + mock_api.get_device_info.return_value = (True, {"device_id": 1, "hostname": "sw"}) + mock_api_cls.return_value = mock_api + mock_import.return_value = {"success": True, "device": MagicMock(), "is_vm": False} + + if mock_rq_job is not None: + mock_conn = MagicMock() + mock_queue = MagicMock() + mock_queue.connection = mock_conn + mock_get_queue.return_value = mock_queue + mock_rqjob_cls.fetch.return_value = mock_rq_job + else: + # Simulate RQ unavailable — get_queue raises, triggers DB fallback + mock_get_queue.side_effect = Exception("RQ unavailable") + + from netbox_librenms_plugin.import_utils.bulk_import import bulk_import_devices_shared + + bulk_import_devices_shared(device_ids, user=MagicMock(), server_key=None, job=job) + + return mock_import.call_count + + def test_rq_stopped_cancels_import_loop(self): + """When RQ job is_stopped, import loop should break early.""" + rq_job = MagicMock() + rq_job.is_stopped = True + rq_job.is_failed = False + rq_job.get_status.return_value = "stopped" + + # With 6 devices and RQ stopped on first check (idx=1), at most 1 device processed + count = self._run_bulk_import(mock_rq_job=rq_job, device_ids=[1, 2, 3, 4, 5, 6]) + assert count == 0 # break before first import + + def test_rq_failed_cancels_import_loop(self): + """When RQ job is_failed, import loop should break early.""" + rq_job = MagicMock() + rq_job.is_stopped = False + rq_job.is_failed = True + rq_job.get_status.return_value = "failed" + + count = self._run_bulk_import(mock_rq_job=rq_job, device_ids=[1, 2, 3, 4, 5, 6]) + assert count == 0 + + def test_rq_unavailable_falls_back_to_db_check(self): + """When RQ is unavailable, DB status check is used as fallback.""" + # mock_rq_job=None triggers the side_effect=Exception path + count = self._run_bulk_import(mock_rq_job=None, db_status="failed", device_ids=[1]) + # With DB status "failed", import should not run + assert count == 0 + + def test_healthy_job_runs_all_devices(self): + """When job is healthy, all devices should be imported.""" + rq_job = MagicMock() + rq_job.is_stopped = False + rq_job.is_failed = False + rq_job.get_status.return_value = "started" + + count = self._run_bulk_import(mock_rq_job=rq_job, device_ids=[1, 2, 3]) + assert count == 3 diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index ac724088d3..855081587d 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -788,7 +788,7 @@ def _build_sync_info(libre_device, existing_device): netbox_platform = platform_info["netbox_platform"] matching_platform = platform_info["matching_platform"] - platform_synced = librenms_os == "-" or ( + platform_synced = librenms_os == "-" or bool( netbox_platform and matching_platform and netbox_platform.pk == matching_platform.pk ) From df133cb261931581605d721769128043609666d9 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 17:49:08 +0100 Subject: [PATCH 28/39] port: upstream PR#227 maintainer commits to inventory branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Template: cluster/rack badge → text+tick icon (0a24a51 completion) - Template: naming preference badges in Device Information card header (d1c015f) - actions.py: _resolve_naming_preferences gains GET support for HTMX hx-include (d1c015f) - actions.py: DeviceValidationDetailsView passes use_sysname/strip_domain to template - actions.py: add _save_device() helper with full_clean()+ValidationError+IntegrityError handling - actions.py: replace all existing_device.save() with _save_device() in conflict actions - device_operations.py: add naming_criteria dict to validate_device_for_import result - device_operations.py: VC-aware name comparison for librenms_id-matched devices - device_operations.py: call update_vc_member_suggested_names when VC is a stack - bulk_import.py: add use_sysname/strip_domain params to process_device_filters - jobs.py: add use_sysname/strip_domain params to FilterDevicesJob.run() - list.py: resolve naming prefs and pass use_sysname/strip_domain to process_device_filters - list.py: pass use_sysname/strip_domain to FilterDevicesJob enqueue - tables/device_status.py: add #use-sysname-toggle, #strip-domain-toggle to all 4 Details hx-include - librenms_import.js: re-init Bootstrap tooltips after modal HTMX content swap (d1c015f) - tests: add TestNameMatchesWithNamingPreferences (8 tests: VC-aware name matching) - tests: fix existing MagicMock devices to set virtual_chassis=None/vc_position=None --- .../import_utils/bulk_import.py | 6 + .../import_utils/device_operations.py | 31 ++- netbox_librenms_plugin/jobs.py | 6 + .../js/librenms_import.js | 6 + .../tables/device_status.py | 8 +- .../htmx/device_validation_details.html | 44 +++- .../tests/test_import_utils.py | 236 ++++++++++++++++++ .../views/imports/actions.py | 55 +++- netbox_librenms_plugin/views/imports/list.py | 40 ++- 9 files changed, 400 insertions(+), 32 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index f943337ab8..b96d2534ab 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -388,6 +388,8 @@ def process_device_filters( job=None, request=None, return_cache_status: bool = False, + use_sysname: bool = True, + strip_domain: bool = False, ) -> List[dict] | tuple[List[dict], bool]: """ Process LibreNMS device filters and return validated devices. @@ -406,6 +408,8 @@ def process_device_filters( job: Optional JobRunner instance for logging job events request: Optional Django request for client disconnect detection (synchronous only) return_cache_status: When True, returns (devices, from_cache) tuple + use_sysname: If True, prefer sysName over hostname for device name resolution + strip_domain: If True, strip domain suffix from device name Returns: List[dict]: Validated devices with _validation key, or tuple of (devices, from_cache) @@ -559,6 +563,8 @@ def _is_active(d): include_vc_detection=vc_detection_enabled, force_vc_refresh=clear_cache, server_key=api.server_key, + use_sysname=use_sysname, + strip_domain=strip_domain, ) except (BrokenPipeError, ConnectionError, IOError) as e: if request: diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index b0a3043686..487c915cf1 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -16,7 +16,12 @@ match_librenms_hardware_to_device_type, ) from .cache import get_import_device_cache_key -from .virtual_chassis import empty_virtual_chassis_data, get_virtual_chassis_data +from .virtual_chassis import ( + _generate_vc_member_name, + empty_virtual_chassis_data, + get_virtual_chassis_data, + update_vc_member_suggested_names, +) logger = logging.getLogger(__name__) @@ -235,6 +240,7 @@ def validate_device_for_import( "rack": None, "available_racks": [], }, + "naming_criteria": None, # Populated after resolved_name is set } try: @@ -248,6 +254,13 @@ def validate_device_for_import( device_id=librenms_id, ) result["resolved_name"] = hostname + result["naming_criteria"] = { + "use_sysname": use_sysname, + "strip_domain": strip_domain, + "raw_sysname": libre_device.get("sysName") or "", + "raw_hostname": libre_device.get("hostname") or "", + "source": "sysname" if use_sysname and libre_device.get("sysName") else "hostname", + } logger.debug( f"Checking for existing device/VM: " f"librenms_id={librenms_id} (type={type(librenms_id).__name__}), " @@ -309,8 +322,19 @@ def validate_device_for_import( if isinstance(existing_device.custom_field_data.get("librenms_id"), int): result["librenms_id_needs_migration"] = True - # Check if name matches resolved name (accounts for use_sysname/strip_domain) - if hostname and existing_device.name == hostname: + # Check if name matches resolved name (VC-aware: compare against VC member name) + if hostname and existing_device.virtual_chassis and existing_device.vc_position: + vc_expected_name = _generate_vc_member_name( + hostname, + existing_device.vc_position, + serial=existing_device.serial or "", + ) + if existing_device.name == vc_expected_name: + result["name_matches"] = True + else: + result["name_sync_available"] = True + result["suggested_name"] = vc_expected_name + elif hostname and existing_device.name == hostname: result["name_matches"] = True elif hostname and existing_device.name != hostname: result["name_sync_available"] = True @@ -586,6 +610,7 @@ def validate_device_for_import( f"Virtual chassis CONFIRMED for device {hostname}: " f"{vc_detection['member_count']} members" ) + result["virtual_chassis"] = update_vc_member_suggested_names(vc_detection, hostname) except Exception as e: logger.exception(f"Exception during VC detection for device {hostname}: {e}") result["virtual_chassis"]["detection_error"] = str(e) diff --git a/netbox_librenms_plugin/jobs.py b/netbox_librenms_plugin/jobs.py index bb3d00e37d..45422ed4f7 100644 --- a/netbox_librenms_plugin/jobs.py +++ b/netbox_librenms_plugin/jobs.py @@ -46,6 +46,8 @@ def run( show_disabled, exclude_existing=False, server_key=None, + use_sysname=True, + strip_domain=False, **kwargs, ): """ @@ -60,6 +62,8 @@ def run( show_disabled: Whether to include disabled devices exclude_existing: Whether to exclude devices that already exist in NetBox server_key: Optional LibreNMS server key for multi-server setups + use_sysname: If True, prefer sysName over hostname + strip_domain: If True, strip domain suffix from device name **kwargs: Additional job parameters """ from netbox_librenms_plugin.import_utils import process_device_filters @@ -88,6 +92,8 @@ def run( show_disabled=show_disabled, exclude_existing=exclude_existing, job=self, + use_sysname=use_sysname, + strip_domain=strip_domain, ) # Store device IDs for result retrieval diff --git a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js index 728406c703..2e130b1284 100644 --- a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js +++ b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js @@ -1094,6 +1094,12 @@ } showModal(modalElement, fallbackBackdropRef); + + // Re-initialize tooltips for newly swapped modal content + if (typeof bootstrap !== 'undefined' && bootstrap.Tooltip) { + const tooltipEls = modalContent.querySelectorAll('[data-bs-toggle="tooltip"]'); + [...tooltipEls].map(el => new bootstrap.Tooltip(el)); + } } document.body.addEventListener('htmx:afterSwap', ensureModalVisible); diff --git a/netbox_librenms_plugin/tables/device_status.py b/netbox_librenms_plugin/tables/device_status.py index 58eeb7c2ad..3f03764cef 100644 --- a/netbox_librenms_plugin/tables/device_status.py +++ b/netbox_librenms_plugin/tables/device_status.py @@ -495,7 +495,7 @@ def render_actions(self, value, record): f'class="btn btn-sm {btn_class}" ' f"{aria_attr}" f'hx-get="{details_url}" ' - f'hx-include="[name=cluster_{device_id}], [name=role_{device_id}], [name=rack_{device_id}]" ' + f'hx-include="[name=cluster_{device_id}], [name=role_{device_id}], [name=rack_{device_id}], #use-sysname-toggle, #strip-domain-toggle" ' f'hx-target="#htmx-modal-content" ' f'hx-swap="innerHTML" ' f'title="{btn_title}">' @@ -517,7 +517,7 @@ def render_actions(self, value, record): f'