From 784072f649ba4da2463ac01ae13ded72ee50e384 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 26 Feb 2026 22:44:18 +0100 Subject: [PATCH 01/27] refactor: split import_utils.py into a package Break the monolithic import_utils.py into focused modules: - permissions.py: user permission checks - cache.py: cache key generation and management - filters.py: device filtering and retrieval from LibreNMS - virtual_chassis.py: VC detection, creation, member management - device_operations.py: device validation, import, and fetch - vm_operations.py: VM creation and bulk import - bulk_import.py: bulk device import orchestration and filter processing The __init__.py re-exports all public names, so existing callers (views, jobs, tests) continue working without import changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- netbox_librenms_plugin/import_utils.py | 2484 ----------------- .../import_utils/__init__.py | 52 + .../import_utils/bulk_import.py | 563 ++++ netbox_librenms_plugin/import_utils/cache.py | 158 ++ .../import_utils/device_operations.py | 859 ++++++ .../import_utils/filters.py | 253 ++ .../import_utils/permissions.py | 48 + .../import_utils/virtual_chassis.py | 440 +++ .../import_utils/vm_operations.py | 216 ++ 9 files changed, 2589 insertions(+), 2484 deletions(-) delete mode 100644 netbox_librenms_plugin/import_utils.py create mode 100644 netbox_librenms_plugin/import_utils/__init__.py create mode 100644 netbox_librenms_plugin/import_utils/bulk_import.py create mode 100644 netbox_librenms_plugin/import_utils/cache.py create mode 100644 netbox_librenms_plugin/import_utils/device_operations.py create mode 100644 netbox_librenms_plugin/import_utils/filters.py create mode 100644 netbox_librenms_plugin/import_utils/permissions.py create mode 100644 netbox_librenms_plugin/import_utils/virtual_chassis.py create mode 100644 netbox_librenms_plugin/import_utils/vm_operations.py diff --git a/netbox_librenms_plugin/import_utils.py b/netbox_librenms_plugin/import_utils.py deleted file mode 100644 index feac6e3e99..0000000000 --- a/netbox_librenms_plugin/import_utils.py +++ /dev/null @@ -1,2484 +0,0 @@ -""" -Utilities for importing devices from LibreNMS to NetBox. - -This module provides functions for: -- Validating LibreNMS devices for import -- Retrieving filtered LibreNMS devices -- Importing single and multiple devices -- Smart matching of NetBox objects -- Permission checking for import operations -""" - -import logging -from typing import List - -from core.choices import JobStatusChoices -from dcim.models import Device, DeviceRole, DeviceType, Rack, Site, VirtualChassis -from django.core.cache import cache -from django.core.exceptions import PermissionDenied -from django.db import transaction -from django.utils import timezone -from virtualization.models import Cluster - -from .librenms_api import LibreNMSAPI -from .utils import ( - find_matching_platform, - find_matching_site, - match_librenms_hardware_to_device_type, -) - -logger = logging.getLogger(__name__) - - -# ============================================================================= -# Permission Check Helpers -# ============================================================================= - - -def check_user_permissions(user, permissions): - """ - Check if user has all required permissions. - - Args: - user: The user object to check permissions for - permissions: List of permission strings (e.g., ['dcim.add_device', 'dcim.add_interface']) - - Returns: - tuple: (has_all_permissions: bool, missing_permissions: list[str]) - - Raises: - PermissionDenied: If user is None (no user context available) - """ - if user is None: - raise PermissionDenied("No user context available for permission check") - - missing = [perm for perm in permissions if not user.has_perm(perm)] - return (len(missing) == 0, missing) - - -def require_permissions(user, permissions, action_description="perform this action"): - """ - Require user has all permissions, raising PermissionDenied if not. - - Args: - user: The user object to check permissions for - permissions: List of permission strings - action_description: Human-readable description for error message - - Raises: - PermissionDenied: If user lacks any required permission - """ - has_perms, missing = check_user_permissions(user, permissions) - if not has_perms: - missing_str = ", ".join(missing) - raise PermissionDenied( - f"You do not have permission to {action_description}. Missing permissions: {missing_str}" - ) - - -def get_cache_metadata_key(server_key: str, filters: dict, vc_enabled: bool) -> str: - """ - Generate a consistent cache metadata key from filter parameters. - - Args: - server_key: LibreNMS server identifier - filters: Filter dictionary - vc_enabled: Whether VC detection is enabled - - Returns: - str: Consistent cache key for metadata - """ - # Sort filter items to ensure consistent key generation - filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v) - return f"librenms_filter_cache_metadata_{server_key}_{filter_parts}_{vc_enabled}" - - -def get_active_cached_searches(server_key: str) -> list[dict]: - """ - Retrieve all active cached searches for a server and enrich with display-friendly values. - - Enriches raw filter IDs with human-readable names by looking up location names - from cached choices and converting type codes to display names. - - Args: - server_key: LibreNMS server identifier - - Returns: - List of dicts containing cache metadata with enriched display_filters - """ - from datetime import datetime, timezone - - cache_index_key = f"librenms_cache_index_{server_key}" - cache_index = cache.get(cache_index_key, []) - - active_searches = [] - valid_cache_keys = [] - - # Get location and type choices for enriching display - location_choices = {} - type_choices = { - "": "All Types", - "network": "Network", - "server": "Server", - "storage": "Storage", - "wireless": "Wireless", - "firewall": "Firewall", - "power": "Power", - "appliance": "Appliance", - "printer": "Printer", - "loadbalancer": "Load Balancer", - "other": "Other", - } - - # Get cached location choices for enrichment - location_cache_key = "librenms_locations_choices" - cached_locations = cache.get(location_cache_key) - if cached_locations: - location_choices = dict(cached_locations) - - for cache_key in cache_index: - metadata = cache.get(cache_key) - if metadata: - # Cache still exists, calculate time remaining - cached_at = datetime.fromisoformat(metadata.get("cached_at")) - cache_timeout = metadata.get("cache_timeout", 300) - now = datetime.now(timezone.utc) - age_seconds = (now - cached_at).total_seconds() - remaining_seconds = max(0, cache_timeout - age_seconds) - - if remaining_seconds > 0: - # Add remaining time and cache key - metadata["remaining_seconds"] = int(remaining_seconds) - metadata["cache_key"] = cache_key - - # Enrich filters with human-readable display values - if "filters" in metadata: - display_filters = metadata["filters"].copy() - # Convert location ID to location name - if "location" in display_filters and display_filters["location"] in location_choices: - display_filters["location"] = location_choices[display_filters["location"]] - # Convert type code to display name - if "type" in display_filters and display_filters["type"] in type_choices: - display_filters["type"] = type_choices[display_filters["type"]] - metadata["display_filters"] = display_filters - else: - # Fallback if filters key missing - metadata["display_filters"] = {} - - active_searches.append(metadata) - valid_cache_keys.append(cache_key) - - # Clean up index if any keys have expired - if len(valid_cache_keys) < len(cache_index): - cache.set(cache_index_key, valid_cache_keys, timeout=3600) - - # Sort by most recent first - active_searches.sort(key=lambda x: x.get("cached_at", ""), reverse=True) - - return active_searches - - -def get_validated_device_cache_key(server_key: str, filters: dict, device_id: int | str, vc_enabled: bool) -> str: - """ - Generate a consistent cache key for validated device data. - - This ensures both synchronous and background job processing use the same - cache keys, avoiding duplicate validation work and cache entries. - - Args: - server_key: LibreNMS server key - filters: Filter dict with location, type, os, hostname, sysname, hardware keys - device_id: LibreNMS device ID - vc_enabled: Whether virtual chassis detection was enabled - - Returns: - str: Cache key for the validated device - - Example: - >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) - >>> key - 'validated_device_default_-1234567890_123_vc' - """ - # Sort filters for consistent hashing - filter_hash = hash(str(sorted(filters.items()))) - vc_part = "vc" if vc_enabled else "novc" - return f"validated_device_{server_key}_{filter_hash}_{device_id}_{vc_part}" - - -def get_import_device_cache_key(device_id: int | str, server_key: str = "default") -> str: - """ - Generate cache key for raw LibreNMS device data. - - This key is used to cache raw device data (without validation metadata) - to avoid redundant API calls when users interact with dropdowns during - the import workflow. - - Args: - device_id: LibreNMS device ID - server_key: LibreNMS server identifier for multi-server setups - - Returns: - str: Cache key for the device data - - Example: - >>> get_import_device_cache_key(123, "production") - 'import_device_data_production_123' - """ - return f"import_device_data_{server_key}_{device_id}" - - -def _determine_device_name( - libre_device: dict, - use_sysname: bool = True, - strip_domain: bool = False, - device_id: int | str = None, -) -> str: - """ - Determine the device/VM name from LibreNMS data. - - Centralized logic for building device names with consistent handling of: - - sysName vs hostname preference - - Domain stripping (avoiding IP addresses) - - Fallback to device_id when name is missing - - Args: - libre_device: Device data from LibreNMS - use_sysname: If True, prefer sysName; if False, use hostname - strip_domain: If True, strip domain suffix (e.g., '.example.com') - device_id: LibreNMS device ID for fallback name generation - - Returns: - str: The determined device name - - Example: - >>> _determine_device_name({'sysName': 'router.example.com', 'hostname': 'router'}, - ... use_sysname=True, strip_domain=True) - 'router' - """ - # Determine base name based on use_sysname preference - if use_sysname: - name = libre_device.get("sysName") or libre_device.get("hostname") - else: - name = libre_device.get("hostname") or libre_device.get("sysName") - - # Fallback to device_id if no name found - if not name: - if device_id is not None: - name = f"device-{device_id}" - else: - name = libre_device.get("device_id", "unknown") - name = f"device-{name}" - - # Strip domain if requested (but not for IP addresses) - if strip_domain and name and "." in name: - try: - from ipaddress import ip_address - - ip_address(name) - # It's a valid IP address, don't strip - except ValueError: - # Not an IP, safe to strip domain - name = name.split(".")[0] - - return name - - -def empty_virtual_chassis_data() -> dict: - """Public helper for callers that need a blank VC payload.""" - - return { - "is_stack": False, - "member_count": 0, - "members": [], - "detection_error": None, - } - - -def _clone_virtual_chassis_data(data: dict | None) -> dict: - """Return a defensive copy of cached VC data to avoid shared references.""" - - if not data: - return empty_virtual_chassis_data() - - members = [] - for idx, member in enumerate(data.get("members", [])): - member_copy = member.copy() - raw_position = member_copy.get("position", idx) - try: - member_copy["position"] = int(raw_position) - except (TypeError, ValueError): - member_copy["position"] = idx - members.append(member_copy) - - member_count = data.get("member_count") or len(members) - - return { - "is_stack": bool(data.get("is_stack")), - "member_count": member_count, - "members": members, - "detection_error": data.get("detection_error"), - } - - -_VC_CACHE_VERSION = "v1" - - -def _vc_cache_key(api: LibreNMSAPI, device_id: int | str) -> str: - server_key = getattr(api, "server_key", "default") - return f"librenms_vc_detection_{_VC_CACHE_VERSION}_{server_key}_{device_id}" - - -def get_virtual_chassis_data(api: LibreNMSAPI, device_id: int | str, *, force_refresh: bool = False) -> dict: - """Fetch (and cache) virtual chassis data for a LibreNMS device.""" - - if not api or device_id is None: - return empty_virtual_chassis_data() - - cache_key = _vc_cache_key(api, device_id) - if not force_refresh: - cached = cache.get(cache_key) - if cached is not None: - return _clone_virtual_chassis_data(cached) - - detection_data = detect_virtual_chassis_from_inventory(api, device_id) - if detection_data and "detection_error" not in detection_data: - detection_data["detection_error"] = None - - cache_value = _clone_virtual_chassis_data(detection_data) if detection_data else empty_virtual_chassis_data() - - cache_timeout = getattr(api, "cache_timeout", 300) or 300 - cache.set(cache_key, cache_value, timeout=cache_timeout) - return _clone_virtual_chassis_data(cache_value) - - -def prefetch_vc_data_for_devices(api: LibreNMSAPI, device_ids: List[int], *, force_refresh: bool = False) -> None: - """ - Pre-warm the virtual chassis cache for multiple devices. - - This eliminates the 0.5-1s delay when rendering the import table - by proactively fetching VC data before validation. - - Args: - api: LibreNMSAPI instance - device_ids: List of LibreNMS device IDs to prefetch VC data for - force_refresh: When True, bypass cache and fetch fresh data - - Example: - >>> # Before rendering import table - >>> prefetch_vc_data_for_devices(api, [123, 124, 125]) - >>> # Now all validate_device_for_import() calls hit cache instantly - """ - if not api or not device_ids: - return - - logger.debug(f"Pre-warming VC cache for {len(device_ids)} devices") - - for idx, device_id in enumerate(device_ids): - # This populates the cache if empty, or skips if already cached - try: - get_virtual_chassis_data(api, device_id, force_refresh=force_refresh) - except (BrokenPipeError, ConnectionError, IOError, OSError) as e: - logger.warning(f"Connection error during VC prefetch at device {idx}: {e}") - # Stop processing if connection is broken - return - except Exception as e: - # Log but continue for other errors - logger.warning(f"Error prefetching VC data for device {device_id}: {e}") - - logger.debug(f"VC cache warming complete for {len(device_ids)} devices") - - -def get_device_count_for_filters( - api: LibreNMSAPI, - filters: dict, - clear_cache: bool = False, - show_disabled: bool = True, -) -> int: - """ - Get count of LibreNMS devices matching filters. - - This is a lightweight function to determine device count for background job - decision making. Uses the same caching as get_librenms_devices_for_import(). - - Args: - api: LibreNMS API client instance - filters: Filter dict with location, type, os, hostname, sysname keys - clear_cache: Whether to force cache refresh - show_disabled: Whether to include disabled devices - - Returns: - int: Count of devices matching filters - """ - devices = get_librenms_devices_for_import(api, filters=filters, force_refresh=clear_cache) - - # Filter out disabled devices if requested - if not show_disabled: - devices = [d for d in devices if d.get("status") == 1] - - return len(devices) - - -def get_librenms_devices_for_import( - api: LibreNMSAPI = None, - filters: dict = None, - server_key: str = None, - *, - force_refresh: bool = False, - return_cache_status: bool = False, -) -> List[dict] | tuple[List[dict], bool]: - """ - Retrieve LibreNMS devices based on filters. - - Args: - api: LibreNMSAPI instance (if not provided, creates one with server_key) - filters: Dict containing filter parameters: - - location: LibreNMS location/site filter - - type: Device type filter - - os: Operating system filter - - hostname: Hostname filter (partial match) - - sysname: System name filter (partial match) - - status: Device status filter (1=up, 0=down) - - disabled: Include disabled devices (0=active only, 1=all) - server_key: Key for specific server configuration (used if api not provided) - force_refresh: When True, bypass the cache and fetch fresh data - return_cache_status: When True, returns (devices, from_cache) tuple - - Returns: - List of device dictionaries from LibreNMS, or tuple of (devices, from_cache) - if return_cache_status is True. from_cache=True means data was loaded from - existing cache; from_cache=False means data was just fetched from LibreNMS. - """ - try: - # Use provided API instance or create a new one - if api is None: - api = LibreNMSAPI(server_key=server_key) - - # Build LibreNMS API filters using the type/query format - # LibreNMS API v0 expects ?type=X&query=Y format, not direct parameters - # NOTE: API only supports ONE type/query pair, so we'll use the most - # specific filter for the API and apply others client-side - api_filters = {} - client_filters = {} # Filters to apply after fetching from API - - if filters: - # Check for status filter first - it has special handling - if filters.get("status") is not None: - # Status filter uses special types that don't need query param - if filters["status"] == 1: - api_filters["type"] = "up" - elif filters["status"] == 0: - api_filters["type"] = "down" - - # Save ALL other filters for client-side filtering when status is used - if filters.get("location"): - client_filters["location"] = filters["location"] - if filters.get("type"): - client_filters["type"] = filters["type"] - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - else: - # Priority order for type/query filters: location > type > os > hostname > sysname - # Note: When sysname is combined with other filters, it's applied client-side for partial matching - # When sysname is alone, it uses API exact match (type=sysName) - # Note: hardware is always applied client-side for partial matching - # Use first available for API, save others for client-side filtering - if filters.get("location"): - api_filters["type"] = "location_id" - api_filters["query"] = filters["location"] - # Save remaining filters for client-side - if filters.get("type"): - client_filters["type"] = filters["type"] - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("type"): - api_filters["type"] = "type" - api_filters["query"] = filters["type"] - # Save remaining filters for client-side - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("os"): - api_filters["type"] = "os" - api_filters["query"] = filters["os"] - # Save remaining filters for client-side - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("hostname"): - api_filters["type"] = "hostname" - api_filters["query"] = filters["hostname"] - # Save sysname and hardware for client-side - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("sysname"): - # sysname-only filter: Use API exact match (type=sysName&query=) - # This is safe - returns empty if no exact match found - api_filters["type"] = "sysName" - api_filters["query"] = filters["sysname"] - # Save hardware for client-side - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("hardware"): - # hardware-only filter: apply client-side for partial matching - client_filters["hardware"] = filters["hardware"] - - # Note: disabled filter isn't directly supported by LibreNMS API - # We'll filter client-side if needed - - # Use caching to avoid repeated API calls - # Include both API and client filters in cache key - cache_key = f"librenms_devices_import_{server_key}_{hash(str(api_filters))}_{hash(str(client_filters))}" - from_cache = False - - if force_refresh: - cache.delete(cache_key) - else: - cached_result = cache.get(cache_key) - if cached_result is not None: - # No need to deepcopy - cached data isn't mutated - devices = cached_result - from_cache = True - if return_cache_status: - return devices, from_cache - return devices - - success, devices = api.list_devices(api_filters if api_filters else None) - - if not success: - logger.error(f"Failed to retrieve devices from LibreNMS: {devices}") - if return_cache_status: - return [], False - return [] - - # Apply client-side filters if any - if client_filters: - devices = _apply_client_filters(devices, client_filters) - - # Cache using configured timeout (default 300s) - # No need to deepcopy - Django's cache backend handles serialization - cache.set(cache_key, devices, timeout=api.cache_timeout) - - if return_cache_status: - return devices, from_cache - return devices - - except Exception: - logger.exception("Error retrieving LibreNMS devices for import") - if return_cache_status: - return [], False - return [] - - -def _apply_client_filters(devices: List[dict], filters: dict) -> List[dict]: - """ - Apply client-side filters to device list. - - Args: - devices: List of device dicts from LibreNMS - filters: Dict of filters to apply (location, type, os, hostname, sysname) - - Returns: - Filtered list of devices - """ - filtered = devices - - if filters.get("location"): - location_id = str(filters["location"]) - filtered = [d for d in filtered if str(d.get("location_id", "")) == location_id] - - if filters.get("type"): - device_type = filters["type"].lower() - filtered = [d for d in filtered if d.get("type", "").lower() == device_type] - - if filters.get("os"): - os_filter = filters["os"].lower() - filtered = [d for d in filtered if os_filter in d.get("os", "").lower()] - - if filters.get("hostname"): - hostname_filter = filters["hostname"].lower() - filtered = [d for d in filtered if hostname_filter in d.get("hostname", "").lower()] - - if filters.get("sysname"): - sysname_filter = filters["sysname"].lower() - filtered = [d for d in filtered if sysname_filter in d.get("sysName", "").lower()] - - if filters.get("hardware"): - hardware_filter = filters["hardware"].lower() - filtered = [d for d in filtered if hardware_filter in (d.get("hardware") or "").lower()] - - return filtered - - -def validate_device_for_import( - libre_device: dict, - import_as_vm: bool = False, - api: "LibreNMSAPI" = None, - *, - include_vc_detection: bool = True, - force_vc_refresh: bool = False, -) -> dict: - """ - Validate if a LibreNMS device can be imported to NetBox. - - Performs comprehensive validation: - - Checks if device already exists in NetBox - - Validates required prerequisites (Site, DeviceType, DeviceRole for devices) - OR (Cluster for VMs) - - Provides smart matching for missing objects - - Detects virtual chassis/stack configuration (if API provided) - - Returns detailed validation status - - Args: - libre_device: Device data from LibreNMS - import_as_vm: If True, validate for VM import instead of device import - api: Optional LibreNMSAPI instance for virtual chassis detection - include_vc_detection: Skip VC detection when False to speed up bulk operations - force_vc_refresh: When True, bypass cached VC data and re-query LibreNMS - - Returns: - dict: Validation result with structure: - { - 'is_ready': bool, # Can import without user intervention - 'can_import': bool, # Can import (possibly after configuration) - 'import_as_vm': bool, # Whether importing as VM - 'existing_device': Device or VirtualMachine or None, - 'issues': List[str], # Blocking issues - 'warnings': List[str], # Non-blocking warnings - 'site': { # Only for devices - 'found': bool, - 'site': Site or None, - 'match_type': str, # 'exact' or None - 'suggestions': List[Site] # Alternative suggestions - }, - 'device_type': { # Only for devices - 'found': bool, - 'device_type': DeviceType or None, - 'match_type': str, # 'exact' or None - 'suggestions': List[dict] # Device types for user selection - }, - 'device_role': { # Only for devices - 'found': bool, # Always False - requires manual selection - 'role': DeviceRole or None, - 'available_roles': List[DeviceRole] # All roles for user selection - }, - 'cluster': { # Only for VMs - 'found': bool, # Always False - requires manual selection - 'cluster': Cluster or None, - 'available_clusters': List[Cluster] # All clusters for user selection - }, - 'platform': { - 'found': bool, - 'platform': Platform or None, - 'match_type': str # 'exact' or None - } - } - - Example: - >>> validation = validate_device_for_import(libre_device) - >>> if validation['is_ready']: - ... import_single_device(libre_device['device_id']) - """ - result = { - "is_ready": False, - "can_import": False, - "import_as_vm": import_as_vm, - "existing_device": None, - "existing_match_type": None, # Track how existing device was matched - "serial_action": None, # None, "link", "conflict", "update_serial", "hostname_differs" - "serial_confirmed": False, # True when librenms_id match and serial matches - "serial_duplicate": False, # True when incoming serial is already on a different device - "name_matches": False, # True when existing device name matches LibreNMS sysName - "name_sync_available": False, # True when existing device name differs from sysName - "suggested_name": None, # sysName to suggest when name_sync_available is True - "device_type_mismatch": False, # True when existing device's type differs from LibreNMS - "issues": [], - "warnings": [], - "virtual_chassis": empty_virtual_chassis_data(), - "site": { - "found": False, - "site": None, - "match_type": None, - "suggestions": [], - }, - "device_type": { - "found": False, - "device_type": None, - "match_type": None, - "suggestions": [], - }, - "device_role": { - "found": False, - "role": None, - "available_roles": [], - }, - "cluster": { - "found": False, - "cluster": None, - "available_clusters": [], - }, - "platform": {"found": False, "platform": None, "match_type": None}, - "rack": { - "found": False, - "rack": None, - "available_racks": [], - }, - } - - try: - # 1. Check if device/VM already exists in NetBox - # Always check both Devices AND VMs to properly detect existing objects - librenms_id = libre_device.get("device_id") - hostname = libre_device.get("hostname", "") - logger.debug( - f"Checking for existing device/VM: " - f"librenms_id={librenms_id} (type={type(librenms_id).__name__}), " - f"hostname={hostname}" - ) - - from virtualization.models import VirtualMachine - - # Check for existing VM first (by librenms_id custom field) - # Always query with int to match custom field type - try: - existing_vm = VirtualMachine.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() - except (ValueError, TypeError): - # librenms_id is not convertible to int; no match will be found - existing_vm = None - - if existing_vm: - logger.info(f"Found existing VM: {existing_vm.name} (matched by librenms_id={librenms_id})") - result["existing_device"] = existing_vm - result["existing_match_type"] = "librenms_id" - result["import_as_vm"] = True # Force VM mode since VM exists - result["can_import"] = False - - # Check if name matches sysName - # Note: name_sync_available/suggested_name are intentionally not set for VMs - # because UpdateDeviceNameView only supports Device objects; VM name-sync - # would require a separate implementation. - sys_name = libre_device.get("sysName") or "" - if sys_name and existing_vm.name == sys_name: - result["name_matches"] = True - - # Check for existing Device (by librenms_id custom field) - # Always query with int to match custom field type - if not result["existing_device"]: - try: - existing_device = Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() - except (ValueError, TypeError): - # librenms_id is not convertible to int; no match will be found - existing_device = None - - if existing_device: - logger.info(f"Found existing device: {existing_device.name} (matched by librenms_id={librenms_id})") - result["existing_device"] = existing_device - result["existing_match_type"] = "librenms_id" - result["can_import"] = False - - # Check if name matches sysName - sys_name = libre_device.get("sysName") or "" - if sys_name and existing_device.name == sys_name: - result["name_matches"] = True - elif sys_name and existing_device.name != sys_name: - result["name_sync_available"] = True - result["suggested_name"] = sys_name - - # Check for serial drift on the linked device - incoming_serial = libre_device.get("serial") or "" - if incoming_serial and incoming_serial != "-": - if existing_device.serial and existing_device.serial == incoming_serial: - result["serial_confirmed"] = True - elif existing_device.serial and existing_device.serial != incoming_serial: - serial_conflict = ( - Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() - ) - if serial_conflict: - result["serial_action"] = "conflict" - result["serial_duplicate"] = True - result["warnings"].append( - f"Serial conflict: incoming serial '{incoming_serial}' is already assigned to " - f"device '{serial_conflict.name}' (ID: {serial_conflict.pk}) in NetBox. " - f"Investigate which device should own this serial before updating." - ) - else: - result["serial_action"] = "update_serial" - result["warnings"].append( - f"Serial number differs (NetBox: '{existing_device.serial}', " - f"LibreNMS: '{incoming_serial}'). Hardware may have been replaced." - ) - - # Only check hostname/serial/IP if not already matched by librenms_id - if not result["existing_device"]: - # Check by hostname/name - Check both VMs and Devices for conflicts - existing_vm = VirtualMachine.objects.filter(name__iexact=hostname).first() - existing_device = Device.objects.filter(name__iexact=hostname).first() - - # If BOTH exist with same hostname, it's ambiguous - don't match either - if existing_vm and existing_device: - logger.warning( - f"Hostname conflict: Both VM '{existing_vm.name}' and Device " - f"'{existing_device.name}' exist with hostname '{hostname}'" - ) - result["warnings"].append( - f"Both a VM and Device exist with hostname '{hostname}' in NetBox. " - f"Cannot determine which to match. Please set the librenms_id custom field on the correct object." - ) - # Don't set existing_device, don't block import - let user proceed as new - # This allows them to import and then resolve the conflict manually - elif existing_vm: - logger.info(f"Found existing VM by hostname: {existing_vm.name}") - result["existing_device"] = existing_vm - result["existing_match_type"] = "hostname" - result["import_as_vm"] = True # Force VM mode since VM exists - result["warnings"].append( - f"VM with same hostname exists in NetBox as '{existing_vm.name}' (not linked to LibreNMS)" - ) - result["can_import"] = False - elif existing_device: - logger.info(f"Found existing device by hostname: {existing_device.name}") - result["existing_device"] = existing_device - result["existing_match_type"] = "hostname" - - # Check for serial conflict on hostname-matched device - incoming_serial = libre_device.get("serial") or "" - if incoming_serial and incoming_serial != "-" and existing_device.serial != incoming_serial: - serial_conflict = ( - Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() - ) - if serial_conflict: - result["serial_action"] = "conflict" - result["serial_duplicate"] = True - result["warnings"].append( - f"Serial conflict: incoming serial '{incoming_serial}' is already assigned to " - f"device '{serial_conflict.name}' (ID: {serial_conflict.pk}) in NetBox. " - f"Investigate which device should own this serial before importing." - ) - else: - result["serial_action"] = "update_serial" - result["warnings"].append( - f"Hostname matches but serial differs (NetBox: '{existing_device.serial}', " - f"LibreNMS: '{incoming_serial}'). Hardware may have been replaced." - ) - else: - result["warnings"].append( - f"Device with same hostname exists in NetBox as '{existing_device.name}' (not linked to LibreNMS)" - ) - - result["can_import"] = False - - # Check by serial number (strong physical match - hardware identity) - if not result["existing_device"]: - serial = libre_device.get("serial") or "" - if serial and serial != "-" and not import_as_vm: - existing_by_serial = Device.objects.filter(serial=serial).first() - if existing_by_serial: - logger.info(f"Found existing device by serial: {existing_by_serial.name} (serial={serial})") - result["existing_device"] = existing_by_serial - result["existing_match_type"] = "serial" - result["can_import"] = False - - if existing_by_serial.name and existing_by_serial.name.lower() == hostname.lower(): - result["warnings"].append( - f"Device with same serial and hostname exists as '{existing_by_serial.name}' " - f"(not linked to LibreNMS)" - ) - result["serial_action"] = "link" - else: - result["warnings"].append( - f"Device with same serial ({serial}) exists as '{existing_by_serial.name}' " - f"but hostname differs (LibreNMS: '{hostname}'). Device may have been reinstalled." - ) - result["serial_action"] = "hostname_differs" - - # Check by primary IP (weaker match, IP could be reassigned) - only for devices - if not result["existing_device"]: - primary_ip = libre_device.get("ip") - if primary_ip and not import_as_vm: - from ipam.models import IPAddress - - existing_ip = IPAddress.objects.filter(address__net_host=primary_ip).first() - if existing_ip and existing_ip.assigned_object: - device = ( - existing_ip.assigned_object.device - if hasattr(existing_ip.assigned_object, "device") - else None - ) - if device: - result["existing_device"] = device - result["existing_match_type"] = "primary_ip" - result["warnings"].append( - f"IP address {primary_ip} already assigned to device '{device.name}' (not linked to LibreNMS)" - ) - result["can_import"] = False - - # Validate based on import type (Device or VM) - if import_as_vm: - # 2. For VMs: Validate Cluster (required) - Must be manually selected - from virtualization.models import Cluster - - result["cluster"]["found"] = False - result["issues"].append("Cluster must be manually selected before importing as VM") - # Provide list of available clusters for user selection (cached) - cache_key = "librenms_import_all_clusters" - all_clusters = cache.get(cache_key) - if all_clusters is None: - all_clusters = list(Cluster.objects.all()) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, all_clusters, cache_timeout) - result["cluster"]["available_clusters"] = all_clusters - - # Skip device-specific validations for VMs - result["site"]["found"] = True # Not required for VMs - result["device_type"]["found"] = True # Not required for VMs - result["device_role"]["found"] = True # Not required for VMs - - else: - # 2. For Devices: Validate Site (required) - location = libre_device.get("location", "") - site_match = find_matching_site(location) - result["site"] = site_match - - if not site_match["found"]: - result["issues"].append(f"No matching site found for location: '{location}'") - # Get alternative suggestions - if location: - all_sites = Site.objects.all()[:10] # Limit for performance - result["site"]["suggestions"] = list(all_sites) - - # 3. Validate DeviceType (required) - hardware = libre_device.get("hardware", "") - dt_match = match_librenms_hardware_to_device_type(hardware) - result["device_type"] = dt_match - - if not dt_match["matched"]: - result["issues"].append(f"No matching device type found for hardware: '{hardware}'") - # Get some device types for user to choose from - all_device_types = DeviceType.objects.all()[:10] - result["device_type"]["suggestions"] = [ - { - "device_type": dt, - "similarity": 0.0, # No fuzzy matching, just showing options - "match_field": None, - } - for dt in all_device_types - ] - else: - # Rename 'matched' to 'found' for consistency - result["device_type"]["found"] = dt_match["matched"] - result["device_type"]["device_type"] = dt_match["device_type"] - result["device_type"]["match_type"] = dt_match["match_type"] - - # 4. DeviceRole (required) - Must be manually selected by user - logger.debug(f"[{hostname}] Issues BEFORE adding role issue: {result['issues']}") - result["device_role"]["found"] = False - result["issues"].append("Device role must be manually selected before import") - logger.debug(f"[{hostname}] Issues AFTER adding role issue: {result['issues']}") - # Provide list of available roles for user selection (cached) - cache_key = "librenms_import_all_roles" - all_roles = cache.get(cache_key) - if all_roles is None: - all_roles = list(DeviceRole.objects.all()) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, all_roles, cache_timeout) - result["device_role"]["available_roles"] = all_roles - - # 4b. Rack (optional) - Provide available racks for the matched site - if site_match["found"] and site_match["site"]: - site = site_match["site"] - # Use cache to optimize rack lookups per site - cache_key = f"librenms_import_racks_site_{site.pk}" - available_racks = cache.get(cache_key) - - if available_racks is None: - from dcim.models import Rack - from django.db.models import Q - - # Query racks for this site - include both: - # 1. Racks assigned to locations within the site - # 2. Racks directly assigned to the site (without location) - available_racks = list( - Rack.objects.filter(Q(location__site=site) | Q(site=site)) - .select_related("location", "site") - .order_by("location__name", "name") - ) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, available_racks, cache_timeout) - - result["rack"]["available_racks"] = available_racks - # Rack is optional, don't add to issues - result["rack"]["found"] = True # Mark as "found" even if None (optional field) - - # Skip VM-specific validations for devices - result["cluster"]["found"] = True # Not required for devices - - # 5. Match Platform (optional - same for both devices and VMs) - os = libre_device.get("os", "") - platform_match = find_matching_platform(os) - result["platform"] = platform_match - - if not platform_match["found"] and os: - result["warnings"].append(f"No matching platform found for OS: '{os}'") - - # 6. Additional validations - if not hostname: - result["issues"].append("Device has no hostname") - - # 7. Virtual chassis detection (only for devices, not VMs) - if include_vc_detection and not import_as_vm and api is not None: - device_id = libre_device.get("device_id") - if device_id: - try: - logger.debug(f"Calling get_virtual_chassis_data for device {device_id}") - vc_detection = get_virtual_chassis_data(api, device_id, force_refresh=force_vc_refresh) - logger.debug( - f"VC detection result: is_stack={vc_detection.get('is_stack')}, " - f"member_count={vc_detection.get('member_count')}, " - f"members={len(vc_detection.get('members', []))}" - ) - if vc_detection: - result["virtual_chassis"] = vc_detection - if vc_detection["is_stack"]: - logger.debug( - f"Virtual chassis CONFIRMED for device {hostname}: " - f"{vc_detection['member_count']} members" - ) - except Exception as e: - logger.exception(f"Exception during VC detection for device {hostname}: {e}") - result["virtual_chassis"]["detection_error"] = str(e) - else: - logger.debug(f"No device_id found for {hostname}") - - # 8. Determine if device/VM is ready to import - if result["existing_device"]: - # Already matched - can_import was already set to False - result["is_ready"] = False - # Populate role from existing device so the modal shows it - existing = result["existing_device"] - if hasattr(existing, "role") and existing.role: - result["device_role"]["found"] = True - result["device_role"]["role"] = existing.role - - # Check for device type mismatch between existing device and LibreNMS - if hasattr(existing, "device_type") and existing.device_type: - librenms_dt = result["device_type"].get("device_type") - if librenms_dt and existing.device_type.pk != librenms_dt.pk: - result["device_type_mismatch"] = True - result["warnings"].append( - f"Device type mismatch: NetBox has '{existing.device_type}' " - f"but LibreNMS reports '{librenms_dt}'. " - f"This may indicate the wrong device was matched." - ) - else: - result["can_import"] = len(result["issues"]) == 0 - - if import_as_vm: - # For VMs: only cluster is required - result["is_ready"] = result["can_import"] and result["cluster"]["found"] - else: - # For Devices: site, device_type, and device_role are required - result["is_ready"] = ( - result["can_import"] - and result["site"]["found"] - and result["device_type"]["found"] - and result["device_role"]["found"] - ) - - logger.debug( - f"Validation for {libre_device.get('hostname')} ({'VM' if import_as_vm else 'Device'}): " - f"issues={len(result['issues'])}, can_import={result['can_import']}, " - f"issues_list={result['issues']}" - ) - - return result - - except Exception as e: - logger.exception(f"Error validating device for import: {libre_device.get('hostname', 'unknown')}") - result["issues"].append(f"Validation error: {str(e)}") - return result - - -def import_single_device( - device_id: int, - server_key: str = None, - validation: dict = None, - manual_mappings: dict = None, - sync_options: dict = None, - libre_device: dict = None, -) -> dict: - """ - Import a single LibreNMS device to NetBox. - - Args: - device_id: LibreNMS device ID - server_key: LibreNMS server configuration key - validation: Pre-computed validation dict (optional) - manual_mappings: Manual object mappings (optional): - - site_id: NetBox Site ID - - device_type_id: NetBox DeviceType ID - - device_role_id: NetBox DeviceRole ID - - platform_id: NetBox Platform ID (optional) - - rack_id: NetBox Rack ID (optional) - sync_options: Sync options (optional): - - sync_interfaces: bool (default True) - - sync_cables: bool (default True) - - sync_ips: bool (default True) - - sync_fields: bool (default True) - libre_device: Pre-fetched LibreNMS device data (optional). - If provided, skips API call to fetch device info. - - Returns: - dict: Import result with structure: - { - 'success': bool, - 'device': Device object or None, - 'message': str, - 'error': str or None, - 'synced': { - 'interfaces': int, - 'cables': int, - 'ip_addresses': int - } - } - """ - try: - api = LibreNMSAPI(server_key=server_key) - - # Use pre-fetched device data if provided, otherwise fetch from API - if libre_device is None: - success, libre_device = api.get_device_info(device_id) - if not success or not libre_device: - return { - "success": False, - "device": None, - "message": "", - "error": f"Failed to retrieve device {device_id} from LibreNMS", - "synced": {}, - } - - # Validate device if validation not provided - if validation is None: - validation = validate_device_for_import(libre_device) - - # Check if device already exists - if validation.get("existing_device"): - return { - "success": False, - "device": validation["existing_device"], - "message": "", - "error": f"Device already exists: {validation['existing_device'].name}", - "synced": {}, - } - - # Use validation-derived matches, allow manual mappings to override specific fields - site = validation["site"].get("site") - device_type = validation["device_type"].get("device_type") - device_role = validation["device_role"].get("role") - platform = validation["platform"].get("platform") - rack = validation.get("rack", {}).get("rack") - - if manual_mappings: - site = Site.objects.filter(id=manual_mappings.get("site_id")).first() or site - device_type = DeviceType.objects.filter(id=manual_mappings.get("device_type_id")).first() or device_type - device_role = DeviceRole.objects.filter(id=manual_mappings.get("device_role_id")).first() or device_role - - platform_id = manual_mappings.get("platform_id") - if platform_id: - from dcim.models import Platform - - platform = Platform.objects.filter(id=platform_id).first() or platform - - rack_id = manual_mappings.get("rack_id") - if rack_id: - rack = Rack.objects.select_related("location", "site").filter(id=rack_id).first() or rack - - rack = rack or validation.get("rack", {}).get("rack") - - # Validate required fields - if not site: - return { - "success": False, - "device": None, - "message": "", - "error": "Site is required but not provided", - "synced": {}, - } - if not device_type: - return { - "success": False, - "device": None, - "message": "", - "error": "Device type is required but not provided", - "synced": {}, - } - if not device_role: - return { - "success": False, - "device": None, - "message": "", - "error": "Device role is required but not provided", - "synced": {}, - } - - # Create device in NetBox - with transaction.atomic(): - # Determine device name based on sync options - use_sysname = sync_options.get("use_sysname", True) if sync_options else True - strip_domain = sync_options.get("strip_domain", False) if sync_options else False - - device_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=strip_domain, - device_id=device_id, - ) - - # Generate import timestamp comment - import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") - - device_data = { - "name": device_name, - "site": site, - "device_type": device_type, - "role": device_role, - "status": "active" if libre_device.get("status") == 1 else "offline", - "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - "custom_field_data": {"librenms_id": int(device_id)}, - } - - # Add optional fields - if platform: - device_data["platform"] = platform - - if rack: - device_data["rack"] = rack - - serial = libre_device.get("serial", "") - if serial and serial != "-": - device_data["serial"] = serial - - location_name = libre_device.get("location", "") - if location_name and location_name != "-": - from dcim.models import Location - - # Try to find matching location within the site - location = Location.objects.filter(site=site, name__iexact=location_name).first() - if location: - device_data["location"] = location - - # Create the device - device = Device(**device_data) - device.full_clean() - device.save() - - # Sync additional data based on options - sync_options = sync_options or {} - synced = {"interfaces": 0, "cables": 0, "ip_addresses": 0} - - try: - # Sync interfaces - if sync_options.get("sync_interfaces", True): - # This is simplified - would need proper request context - # For now, just log that it should be done - logger.info(f"Interface sync should be performed for device {device.name}") - - # Sync cables - if sync_options.get("sync_cables", True): - logger.info(f"Cable sync should be performed for device {device.name}") - - # Sync IP addresses - if sync_options.get("sync_ips", True): - logger.info(f"IP address sync should be performed for device {device.name}") - - except Exception as e: - logger.warning(f"Error during post-import sync: {str(e)}") - # Don't fail the import if sync fails - - return { - "success": True, - "device": device, - "message": f"Successfully imported device: {device.name}", - "error": None, - "synced": synced, - } - - except Exception as e: - logger.exception(f"Error importing device {device_id}") - return { - "success": False, - "device": None, - "message": "", - "error": str(e), - "synced": {}, - } - - -def bulk_import_devices_shared( - device_ids: List[int], - server_key: str = None, - sync_options: dict = None, - manual_mappings_per_device: dict = None, - libre_devices_cache: dict = None, - job=None, - user=None, -) -> dict: - """ - Shared function for importing multiple LibreNMS devices to NetBox. - - Used by both synchronous imports and background jobs. Handles per-device error - collection and optional progress logging when job context is provided. - - Args: - device_ids: List of LibreNMS device IDs to import - server_key: LibreNMS server configuration key - sync_options: Sync options to apply to all devices - manual_mappings_per_device: Dict mapping device_id to manual_mappings dict - Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} - libre_devices_cache: Optional dict mapping device_id to pre-fetched device data - to avoid redundant API calls. Example: {123: {...device_data...}} - job: Optional JobRunner instance for progress logging and cancellation checks - user: User performing the import (for permission checks). If job is provided, - user is extracted from job.job.user if not explicitly passed. - - Returns: - dict: Bulk import result with structure: - { - 'total': int, - 'success': List[dict], # Successfully imported devices - 'failed': List[dict], # Failed imports with errors - 'skipped': List[dict], # Skipped devices (already exist, etc.) - 'virtual_chassis_created': int # Number of VCs created - } - - Raises: - PermissionDenied: If user lacks required permissions - - Example: - >>> # Synchronous usage - >>> result = bulk_import_devices_shared([1, 2, 3, 4, 5], user=request.user) - >>> # Background job usage - >>> result = bulk_import_devices_shared([1, 2, 3], job=self) - """ - # Extract user from job if not explicitly provided - if user is None and job is not None: - user = getattr(job.job, "user", None) - - # Check permissions at start of bulk operation - required_perms = [ - "dcim.add_device", - "dcim.add_interface", - "dcim.add_virtualchassis", - ] - require_permissions(user, required_perms, "import devices") - - total = len(device_ids) - success_list = [] - failed_list = [] - skipped_list = [] - vc_created_count = 0 - processed_vc_domains = set() # Track VCs already created by domain - - # Initialize API client once for all devices to avoid repeated config parsing - api = LibreNMSAPI(server_key=server_key) - - for idx, device_id in enumerate(device_ids, start=1): - # Check for job cancellation every 5 devices - if job and idx % 5 == 0: - # Refresh job from DB to get current status - job.job.refresh_from_db() - job_status = job.job.status - status_value = job_status.value if hasattr(job_status, "value") else job_status - if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): - if job.logger: - job.logger.warning(f"Import job cancelled at device {idx} of {total}") - else: - logger.warning(f"Import cancelled at device {idx} of {total}") - break - # Log progress - if job.logger: - job.logger.info(f"Imported device {idx} of {total}") - - try: - # Use cached device data if available to avoid redundant API calls - if libre_devices_cache and device_id in libre_devices_cache: - libre_device = libre_devices_cache[device_id] - success = True - else: - success, libre_device = api.get_device_info(device_id) - - if not success or not libre_device: - error_msg = f"Failed to retrieve device {device_id} from LibreNMS" - failed_list.append({"device_id": device_id, "error": error_msg}) - if job and job.logger: - job.logger.error(error_msg) - else: - logger.error(error_msg) - continue - - validation = validate_device_for_import(libre_device, api=api) - - # Build manual mappings from validation + any provided overrides - device_mappings = {} - - # Get site and device_type from validation - if validation["site"].get("found") and validation["site"].get("site"): - device_mappings["site_id"] = validation["site"]["site"].id - if validation["device_type"].get("found") and validation["device_type"].get("device_type"): - device_mappings["device_type_id"] = validation["device_type"]["device_type"].id - if validation["platform"].get("found") and validation["platform"].get("platform"): - device_mappings["platform_id"] = validation["platform"]["platform"].id - - # Override with any manual mappings provided for this device - if manual_mappings_per_device and device_id in manual_mappings_per_device: - device_mappings.update(manual_mappings_per_device[device_id]) - - result = import_single_device( - device_id, - server_key=server_key, - sync_options=sync_options, - manual_mappings=device_mappings if device_mappings else None, - libre_device=libre_device, - ) - - if result["success"]: - success_list.append( - { - "device_id": device_id, - "device": result["device"], - "message": result["message"], - } - ) - - # Handle virtual chassis creation for stacks - vc_data = validation.get("virtual_chassis", {}) - if vc_data.get("is_stack", False): - vc_domain = f"librenms-{device_id}" - - # Only create VC if we haven't processed this stack yet - # Add to set BEFORE attempting creation to prevent race condition - if vc_domain not in processed_vc_domains: - processed_vc_domains.add(vc_domain) - try: - vc = create_virtual_chassis_with_members( - result["device"], - vc_data["members"], - libre_device, - ) - vc_created_count += 1 - log_msg = f"Created VC '{vc.name}' during bulk import for device {device_id}" - if job and job.logger: - job.logger.info(log_msg) - else: - logger.info(log_msg) - except Exception as vc_error: - # Remove from set on failure so retry is possible - processed_vc_domains.discard(vc_domain) - warn_msg = f"Failed to create VC for device {device_id}: {vc_error}" - if job and job.logger: - job.logger.warning(warn_msg) - else: - logger.warning(warn_msg) - # Don't fail the import, just log the warning - - elif result.get("device"): # Device exists - skipped_list.append({"device_id": device_id, "reason": result["error"]}) - else: # Failed to import - failed_list.append({"device_id": device_id, "error": result["error"]}) - if job and job.logger: - job.logger.error(f"Failed to import device {device_id}: {result['error']}") - - except Exception as e: - error_msg = f"Unexpected error importing device {device_id}: {str(e)}" - if job and job.logger: - job.logger.error(error_msg, exc_info=True) - else: - logger.exception(f"Unexpected error importing device {device_id}") - failed_list.append({"device_id": device_id, "error": str(e)}) - - return { - "total": total, - "success": success_list, - "failed": failed_list, - "skipped": skipped_list, - "virtual_chassis_created": vc_created_count, - } - - -def bulk_import_devices( - device_ids: List[int], - server_key: str = None, - sync_options: dict = None, - manual_mappings_per_device: dict = None, - libre_devices_cache: dict = None, - user=None, -) -> dict: - """ - Import multiple LibreNMS devices to NetBox (synchronous). - - This is the public API for synchronous imports. For background job usage, - use bulk_import_devices_shared() with a job context. - - Args: - device_ids: List of LibreNMS device IDs to import - server_key: LibreNMS server configuration key - sync_options: Sync options to apply to all devices - manual_mappings_per_device: Dict mapping device_id to manual_mappings dict - Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} - libre_devices_cache: Optional dict mapping device_id to pre-fetched device data - to avoid redundant API calls. Example: {123: {...device_data...}} - user: User performing the import (for permission checks) - - Returns: - dict: Bulk import result with structure: - { - 'total': int, - 'success': List[dict], # Successfully imported devices - 'failed': List[dict], # Failed imports with errors - 'skipped': List[dict], # Skipped devices (already exist, etc.) - 'virtual_chassis_created': int # Number of VCs created - } - - Raises: - PermissionDenied: If user lacks required permissions - """ - return bulk_import_devices_shared( - device_ids=device_ids, - server_key=server_key, - sync_options=sync_options, - manual_mappings_per_device=manual_mappings_per_device, - libre_devices_cache=libre_devices_cache, - job=None, # No job context for synchronous imports - user=user, - ) - - -def get_librenms_device_by_id(api: LibreNMSAPI, device_id: int) -> dict: - """ - Retrieve a single device from LibreNMS by ID. - - Args: - api: LibreNMSAPI instance - device_id: LibreNMS device ID - - Returns: - Device dictionary or None if not found - """ - try: - # Use the dedicated API endpoint to get device by ID - success, device = api.get_device_info(device_id) - if success and device: - return device - - logger.warning(f"Device {device_id} not found in LibreNMS") - return None - except Exception as e: - logger.exception(f"Failed to get device {device_id} from LibreNMS: {e}") - return None - - -def fetch_device_with_cache( - device_id: int, - api: LibreNMSAPI, - server_key: str = None, - libre_devices_cache: dict = None, -) -> dict | None: - """ - Fetch LibreNMS device from cache or API with automatic caching. - - Checks three sources in order: - 1. Pre-fetched cache dict (if provided) - 2. Django cache (Redis/memory) - 3. LibreNMS API (caches result for future use) - - This function consolidates the device fetching pattern used throughout - the import workflow, eliminating code duplication. - - Args: - device_id: LibreNMS device ID to fetch - api: LibreNMSAPI instance for fallback API calls - server_key: Optional server key for multi-server setups (defaults to api.server_key) - libre_devices_cache: Optional pre-fetched device cache dict - - Returns: - Device dict from LibreNMS, or None if not found - - Example: - >>> # Simple usage - >>> libre_device = fetch_device_with_cache(123, api) - >>> if libre_device: - ... print(libre_device['hostname']) - >>> - >>> # With pre-fetched cache dict - >>> cache_dict = {123: {...}, 456: {...}} - >>> libre_device = fetch_device_with_cache(123, api, libre_devices_cache=cache_dict) - """ - # Check pre-fetched cache dict first (fastest) - if libre_devices_cache and device_id in libre_devices_cache: - return libre_devices_cache[device_id] - - # Check Django cache - cache_key = get_import_device_cache_key(device_id, server_key or api.server_key) - libre_device = cache.get(cache_key) - - if not libre_device: - # Fallback to API fetch - libre_device = get_librenms_device_by_id(api, device_id) - if libre_device: - # Cache for future use - cache.set(cache_key, libre_device, timeout=api.cache_timeout) - - return libre_device - - -def create_vm_from_librenms(libre_device: dict, validation: dict, use_sysname: bool = True, role=None): - """ - Create a NetBox VirtualMachine from LibreNMS device data. - - Args: - libre_device: Device data from LibreNMS - validation: Validation result from validate_device_for_import with import_as_vm=True - use_sysname: If True, prefer sysName; if False, use hostname - role: Optional DeviceRole to assign to the VM - - Returns: - Created VirtualMachine instance - - Raises: - Exception if VM cannot be created - """ - from virtualization.models import VirtualMachine - - if not validation["can_import"]: - raise ValueError(f"VM cannot be imported: {', '.join(validation['issues'])}") - - # Extract matched objects from validation - cluster = validation["cluster"]["cluster"] - platform = validation["platform"].get("platform") - - # Determine VM name - use pre-computed name if available (handles strip_domain) - vm_name = libre_device.get("_computed_name") - if not vm_name: - vm_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=False, - device_id=libre_device.get("device_id"), - ) - - # Generate import timestamp comment - import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") - - # Create the VM with librenms_id custom field - vm = VirtualMachine.objects.create( - name=vm_name, - cluster=cluster, - role=role, # Optional VM role - platform=platform, - comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - custom_field_data={"librenms_id": int(libre_device["device_id"])}, - ) - - logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") - return vm - - -def bulk_import_vms( - vm_imports: dict[int, dict[str, int]], - api: LibreNMSAPI, - sync_options: dict = None, - libre_devices_cache: dict = None, - job=None, - user=None, -) -> dict: - """ - Import multiple LibreNMS devices as VMs in NetBox. - - Handles validation, cluster/role assignment, name determination, - and VM creation. Supports both synchronous and background job execution. - - This function consolidates VM import logic that was previously duplicated - in BulkImportDevicesView and ImportDevicesJob, ensuring consistent behavior - across synchronous and background import paths. - - Args: - vm_imports: Dict mapping device_id to {"cluster_id": int, "device_role_id": int} - api: LibreNMSAPI instance for device fetching - sync_options: Optional dict with use_sysname, strip_domain settings - libre_devices_cache: Optional pre-fetched device data cache - job: Optional JobRunner instance for background job logging/cancellation - user: User performing the import (for permission checks). If job is provided, - user is extracted from job.job.user if not explicitly passed. - - Returns: - Dict with keys: - - success: List of {"device_id": int, "device": VM, "message": str} - - failed: List of {"device_id": int, "error": str} - - skipped: List of {"device_id": int, "reason": str} - - Raises: - PermissionDenied: If user lacks required permissions - - Example: - >>> # Synchronous import from view - >>> vm_imports = {123: {"cluster_id": 5, "device_role_id": 2}} - >>> result = bulk_import_vms(vm_imports, api, sync_options, user=request.user) - >>> print(f"Created {len(result['success'])} VMs") - >>> - >>> # Background job import - >>> result = bulk_import_vms(vm_imports, api, sync_options, cache, job=self) - """ - from netbox_librenms_plugin.import_validation_helpers import ( - apply_cluster_to_validation, - apply_role_to_validation, - ) - - # Extract user from job if not explicitly provided - if user is None and job is not None: - user = getattr(job.job, "user", None) - - # Check permissions at start of bulk operation - require_permissions(user, ["virtualization.add_virtualmachine"], "import VMs") - - result = {"success": [], "failed": [], "skipped": []} - vm_ids = list(vm_imports.keys()) - - # Use job logger if available, otherwise standard logger - log = job.logger if job else logger - - for idx, vm_id in enumerate(vm_ids, start=1): - # Check for job cancellation every 5 VMs - if job and idx % 5 == 0: - job.job.refresh_from_db() - job_status = job.job.status - status_value = job_status.value if hasattr(job_status, "value") else job_status - if status_value in ("failed", "errored"): - log.warning(f"Job cancelled at VM {idx} of {len(vm_ids)}") - break - log.info(f"Imported VM {idx} of {len(vm_ids)}") - - try: - # Fetch device data (uses cache helper) - libre_device = fetch_device_with_cache(vm_id, api, api.server_key, libre_devices_cache) - - if not libre_device: - result["failed"].append( - { - "device_id": vm_id, - "error": f"Device {vm_id} not found in LibreNMS", - } - ) - log.error(f"Device {vm_id} not found in LibreNMS") - continue - - # Validate as VM - validation = validate_device_for_import(libre_device, import_as_vm=True, api=api) - - # Check if VM already exists - if validation.get("existing_device"): - result["skipped"].append( - { - "device_id": vm_id, - "reason": f"VM already exists: {validation['existing_device'].name}", - } - ) - log.info(f"VM already exists: {validation['existing_device'].name}") - continue - - # Apply manual cluster and role selections - vm_mappings = vm_imports[vm_id] - cluster_id = vm_mappings.get("cluster_id") - role_id = vm_mappings.get("device_role_id") - - if cluster_id: - cluster = Cluster.objects.filter(id=cluster_id).first() - if cluster: - apply_cluster_to_validation(validation, cluster) - - role = None - if role_id: - role = DeviceRole.objects.filter(id=role_id).first() - if role: - apply_role_to_validation(validation, role, is_vm=True) - - # Determine VM name - use_sysname = sync_options.get("use_sysname", True) if sync_options else True - strip_domain = sync_options.get("strip_domain", False) if sync_options else False - - vm_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=strip_domain, - device_id=vm_id, - ) - - # Update validation with computed name - libre_device["_computed_name"] = vm_name - - # Create VM - vm = create_vm_from_librenms(libre_device, validation, use_sysname=use_sysname, role=role) - - result["success"].append( - { - "device_id": vm_id, - "device": vm, - "message": f"VM {vm.name} created successfully", - } - ) - log.info(f"Successfully imported VM {vm.name} (ID: {vm_id})") - - except Exception as vm_error: - log.error(f"Failed to import VM {vm_id}: {vm_error}", exc_info=True) - result["failed"].append({"device_id": vm_id, "error": str(vm_error)}) - - return result - - -def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> dict: - """ - Detect if device is a stack/Virtual Chassis by analyzing ENTITY-MIB inventory. - Vendor-agnostic using standard hierarchical structure. - - Args: - api: LibreNMSAPI instance - device_id: LibreNMS device ID - - Returns: - dict with structure: - { - 'is_stack': bool, - 'member_count': int, - 'members': [ - { - 'serial': str, - 'position': int, - 'model': str, - 'name': str, - 'index': int, - 'description': str, - 'suggested_name': str # Generated using master device name - } - ] - } - Returns None if not a stack or detection fails. - - Detection Logic: - 1. Check root level (entPhysicalContainedIn=0) for parent container - 2. Find parent index (entPhysicalClass='stack' or 'chassis') - 3. Get children chassis at that parent's index - 4. If multiple chassis found → Stack detected - """ - try: - # Get the master device info to use for naming - success, device_info = api.get_device_info(device_id) - master_name = None - if success and device_info: - master_name = device_info.get("sysName") or device_info.get("hostname") - - # Step 1: Get root level items - success, root_items = api.get_inventory_filtered(device_id, ent_physical_contained_in=0) - - if not success or not root_items: - logger.debug(f"No root inventory items found for device {device_id}") - return None - - # Step 2: Find parent container index - # Could be class="stack" or the main "chassis" - parent_index = None - for item in root_items: - item_class = item.get("entPhysicalClass") - if item_class in ["stack", "chassis"]: - parent_index = item.get("entPhysicalIndex") - logger.debug(f"VC detection: Found parent container at index {parent_index} for device {device_id}") - break - - if not parent_index: - return None - - # Step 3: Get children chassis at next level - success, child_items = api.get_inventory_filtered( - device_id, - ent_physical_class="chassis", - ent_physical_contained_in=parent_index, - ) - - if not success: - return None - - # Filter for chassis only (in case API filter didn't work) - chassis_items = [item for item in (child_items or []) if item.get("entPhysicalClass") == "chassis"] - - # Step 4: Multiple chassis = stack - if len(chassis_items) <= 1: - return None - - # Step 5: Extract member info - members = [] - for idx, chassis in enumerate(chassis_items): - raw_position = chassis.get("entPhysicalParentRelPos", idx) - try: - position = int(raw_position) - except (TypeError, ValueError): - position = idx - member_data = { - "serial": chassis.get("entPhysicalSerialNum", ""), - "position": position, - "model": chassis.get("entPhysicalModelName", ""), - "name": chassis.get("entPhysicalName", ""), - "index": chassis.get("entPhysicalIndex"), - "description": chassis.get("entPhysicalDescr", ""), - } - - # Generate suggested name if we have master name - if master_name: - member_data["suggested_name"] = _generate_vc_member_name(master_name, position + 1) - else: - member_data["suggested_name"] = f"Member-{position + 1}" - - members.append(member_data) - - # Sort by position - members.sort(key=lambda m: m["position"]) - - logger.info(f"Detected stack with {len(members)} members for device {device_id}") - - return {"is_stack": True, "member_count": len(members), "members": members} - - except Exception as e: - logger.exception(f"Error detecting virtual chassis for device {device_id}: {e}") - return None - - -def _generate_vc_member_name(master_name: str, position: int, serial: str = None) -> str: - """ - Generate name for VC member device using configured pattern from settings. - - Args: - master_name: Name of the master/primary device - position: VC position number - serial: Optional serial number of the member device - - Returns: - Generated member device name - - Examples: - pattern="-M{position}" → "switch01-M2" - pattern=" ({position})" → "switch01 (2)" - pattern="-SW{position}" → "switch01-SW2" - pattern=" [{serial}]" → "switch01 [ABC123]" - """ - # Import here to avoid circular dependency - from .models import LibreNMSSettings - - # Get pattern from settings with fallback to default - try: - settings = LibreNMSSettings.objects.first() - pattern = settings.vc_member_name_pattern if settings else "-M{position}" - except Exception as e: - logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") - pattern = "-M{position}" - - # Prepare format variables - format_vars = { - "master_name": master_name, - "position": position, - "serial": serial or "", - } - - # Apply pattern - pattern should be suffix/prefix, not full name - try: - formatted_suffix = pattern.format(**format_vars) - return f"{master_name}{formatted_suffix}" - except KeyError as e: - logger.error(f"Invalid placeholder in VC naming pattern '{pattern}': {e}. Using default.") - return f"{master_name}-M{position}" - - -def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: - """ - Regenerate suggested VC member names using the actual master device name. - - This ensures preview shows accurate names after use_sysname and strip_domain - are applied to the master device name. - - Args: - vc_data: Virtual chassis detection data dict - master_name: The actual name that will be used for master device in NetBox - - Returns: - Updated vc_data dict with corrected suggested_name for each member - """ - if not vc_data or not vc_data.get("is_stack"): - return vc_data - - for idx, member in enumerate(vc_data.get("members", [])): - raw_position = member.get("position", idx) - try: - base_position = int(raw_position) - except (TypeError, ValueError): - base_position = idx - position = base_position + 1 # Convert to 1-based position - member["position"] = base_position - member["suggested_name"] = _generate_vc_member_name(master_name, position, serial=member.get("serial")) - - return vc_data - - -def create_virtual_chassis_with_members(master_device: Device, members_info: list, libre_device: dict): - """ - Create Virtual Chassis and member devices from detection info. - - This function creates a NetBox VirtualChassis with the master device - and all detected member devices, wrapped in a transaction for safety. - - Args: - master_device: The imported device (becomes VC master) - members_info: List of member dicts from VC detection - libre_device: Original LibreNMS device data - - Returns: - VirtualChassis: The created virtual chassis instance - - Raises: - ValidationError: If member count validation fails - IntegrityError: If duplicate serials/names are detected - Exception: For other creation errors - - Example members_info: - [ - {'serial': 'ABC123', 'position': 0, 'model': 'C9300-48U', 'name': 'Switch 1'}, - {'serial': 'ABC124', 'position': 1, 'model': 'C9300-48U', 'name': 'Switch 2'} - ] - """ - - # Store original master device state for rollback - original_master_name = master_device.name - original_vc = master_device.virtual_chassis - original_vc_position = master_device.vc_position - - try: - with transaction.atomic(): - # Rename master device to include position 1 pattern - master_device_new_name = _generate_vc_member_name(original_master_name, 1, serial=master_device.serial) - - # Check if renamed master conflicts with existing device - if Device.objects.filter(name=master_device_new_name).exclude(pk=master_device.pk).exists(): - logger.warning( - f"Cannot rename master to '{master_device_new_name}' - name already exists. " - f"Keeping original name '{original_master_name}'" - ) - master_base_name = original_master_name - else: - master_device.name = master_device_new_name - master_base_name = original_master_name - - # Create VC using original base name - vc_name = master_base_name - vc = VirtualChassis.objects.create( - name=vc_name, - master=master_device, - domain=f"librenms-{libre_device['device_id']}", - ) - - # Update master device - master_device.virtual_chassis = vc - master_device.vc_position = 1 # Master is position 1 - master_device.save() - - # Create member devices for remaining positions - position = 2 # Start at 2 (master is 1) - members_created = 0 - - for member in members_info: - # Skip if this is the master's serial - if member.get("serial") == master_device.serial: - continue - - serial = member.get("serial") - - member_rack = master_device.rack - member_location = master_device.location or ( - member_rack.location if member_rack and member_rack.location else None - ) - - # Check for duplicate serial - if serial and Device.objects.filter(serial=serial).exists(): - logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") - continue - - member_name = _generate_vc_member_name(master_base_name, position, serial=serial) - - # Check for duplicate name - if Device.objects.filter(name=member_name).exists(): - logger.warning(f"Device with name '{member_name}' already exists, skipping VC member creation") - continue - - Device.objects.create( - name=member_name, - device_type=master_device.device_type, - role=master_device.role, - site=master_device.site, - location=member_location, - rack=member_rack, - platform=master_device.platform, - serial=serial, - virtual_chassis=vc, - vc_position=position, - comments=f"VC member (LibreNMS: {member.get('name', 'Unknown')})\n" - f"Auto-created from stack inventory", - ) - members_created += 1 - position += 1 - - # Validate member count - expected_members = len([m for m in members_info if m.get("serial") != master_device.serial]) - if members_created < expected_members: - logger.warning( - f"Created {members_created} members but expected {expected_members}. " - "Some members may have been skipped due to duplicates." - ) - - logger.info( - f"Created Virtual Chassis '{vc.name}' with {vc.members.count()} total members " - f"(1 master + {members_created} additional)" - ) - - return vc - - except Exception as e: - # Rollback master device to original state - logger.error( - f"Virtual Chassis creation failed for device {master_device.name}: {e}. Rolling back master device changes." - ) - master_device.name = original_master_name - master_device.virtual_chassis = original_vc - master_device.vc_position = original_vc_position - master_device.save() - raise - - -def _refresh_existing_device(validation: dict) -> None: - """Refresh existing_device from DB to pick up changes made in NetBox since caching.""" - existing = validation.get("existing_device") - if not existing or not hasattr(existing, "pk"): - return - try: - from dcim.models import Device - from virtualization.models import VirtualMachine - - if validation.get("import_as_vm"): - refreshed = VirtualMachine.objects.filter(pk=existing.pk).first() - else: - refreshed = Device.objects.filter(pk=existing.pk).first() - - if refreshed: - validation["existing_device"] = refreshed - if hasattr(refreshed, "role") and refreshed.role: - validation["device_role"] = {"found": True, "role": refreshed.role} - else: - # Device was deleted since caching — recompute readiness - validation["existing_device"] = None - validation["existing_match_type"] = None - validation["can_import"] = True - if validation.get("import_as_vm"): - validation["is_ready"] = bool( - validation.get("site", {}).get("found") and validation.get("device_role", {}).get("found") - ) - else: - validation["is_ready"] = bool( - validation.get("site", {}).get("found") - and validation.get("device_type", {}).get("found") - and validation.get("device_role", {}).get("found") - ) - except Exception as e: - existing_id = getattr(existing, "pk", "unknown") if existing else "none" - logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") - - -def process_device_filters( - api: LibreNMSAPI, - filters: dict, - vc_detection_enabled: bool, - clear_cache: bool, - show_disabled: bool, - exclude_existing: bool = False, - job=None, - request=None, - return_cache_status: bool = False, -) -> List[dict] | tuple[List[dict], bool]: - """ - Process LibreNMS device filters and return validated devices. - - Shared function used by both synchronous view and background job processing. - Fetches devices, optionally pre-warms VC cache, validates each device, and - caches results for HTMX row updates. - - Args: - api: LibreNMS API client instance - filters: Filter dict with location, type, os, hostname, sysname, hardware keys - vc_detection_enabled: Whether to detect virtual chassis - clear_cache: Whether to force cache refresh - show_disabled: Whether to include disabled devices - exclude_existing: Whether to exclude devices that already exist in NetBox - job: Optional JobRunner instance for logging job events - request: Optional Django request for client disconnect detection (synchronous only) - return_cache_status: When True, returns (devices, from_cache) tuple - - Returns: - List[dict]: Validated devices with _validation key, or tuple of (devices, from_cache) - if return_cache_status is True. from_cache=True means data was loaded from existing - cache; from_cache=False means data was just fetched from LibreNMS. - """ - # Fetch devices from LibreNMS - if job: - job.logger.info(f"Fetching devices with filters: {filters}") - else: - logger.info(f"Fetching devices with filters: {filters}") - - # Always get cache status internally, even if not returning it - # We need it to determine if metadata should be updated - libre_devices, from_cache = get_librenms_devices_for_import( - api, - filters=filters, - force_refresh=clear_cache, - return_cache_status=True, - ) - - # Filter out disabled devices if requested - if not show_disabled: - libre_devices = [d for d in libre_devices if d.get("status") == 1] - - if job: - job.logger.info(f"Found {len(libre_devices)} devices to process") - else: - logger.info(f"Found {len(libre_devices)} devices") - - # Pre-warm VC cache if needed - if vc_detection_enabled and libre_devices: - device_ids = [d["device_id"] for d in libre_devices] - if job: - job.logger.info( - f"Pre-fetching virtual chassis data for {len(device_ids)} devices. This may take some time..." - ) - else: - logger.info(f"Pre-fetching VC data for {len(device_ids)} devices") - - try: - prefetch_vc_data_for_devices(api, device_ids, force_refresh=clear_cache) - if job: - job.logger.info("Virtual chassis data pre-fetch completed") - except (BrokenPipeError, ConnectionError, IOError) as e: - if request: - logger.info(f"Client disconnected during VC prefetch: {e}") - return [] - raise - - # Validate each device - validated_devices = [] - total = len(libre_devices) - api_for_validation = api if vc_detection_enabled else None - - if job: - job.logger.info(f"Starting validation of {total} devices") - # Initial check if job was already terminated before we even started - try: - from django_rq import get_queue - from rq.job import Job as RQJob - - queue = get_queue("default") - rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) - - if rq_job.is_failed or rq_job.is_stopped: - job.logger.warning("Job was already stopped before validation started") - return [] - except Exception: - # Fall back to DB check if RQ check fails - job.job.refresh_from_db() - if job.job.status == JobStatusChoices.STATUS_FAILED: - job.logger.warning("Job was stopped before validation started") - return [] - else: - logger.info(f"Validating {total} devices") - - for idx, device in enumerate(libre_devices, 1): - # Check for job termination or client disconnect periodically - if idx % 5 == 0 or idx == 1: # Check more frequently (every 5 devices + first device) - if job: - # Check if job was terminated via stop API - # CRITICAL: Check the RQ job status in Redis, not just the DB model - # NetBox's stop endpoint marks the RQ job as failed in Redis - try: - from django_rq import get_queue - from rq.job import Job as RQJob - - queue = get_queue("default") - rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) - - # Check if RQ job is in a stopped state - if rq_job.is_failed or rq_job.is_stopped: - job.logger.info( - f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." - ) - return [] - except Exception: - # If we can't check RQ status, fall back to DB status check - job.job.refresh_from_db() - if job.job.status == JobStatusChoices.STATUS_FAILED: - job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") - return [] - elif request: - # Check for client disconnect - try: - if hasattr(request, "META") and request.META.get("wsgi.input"): - pass - except (BrokenPipeError, ConnectionError, IOError): - logger.info(f"Client disconnected during validation at device {idx}") - return [] - - # Drop any cached validation/meta keys before recomputing - device.pop("_validation", None) - - # Generate shared cache key for this validated device - device_id = device["device_id"] - cache_key = get_validated_device_cache_key( - server_key=api.server_key, - filters=filters, - device_id=device_id, - vc_enabled=vc_detection_enabled, - ) - - # Check if we already have cached validation for this device - # (only if not forcing refresh) - if not clear_cache: - cached_device = cache.get(cache_key) - if cached_device: - # Use cached validation - device["_validation"] = cached_device["_validation"] - - # Refresh existing_device from DB to avoid stale data - # (user may have changed role, name, etc. in NetBox) - _refresh_existing_device(device["_validation"]) - - # Apply exclude_existing filter if enabled - if exclude_existing: - validation = device["_validation"] - if validation["existing_device"]: - continue - - validated_devices.append(device) - continue - - # Not in cache or forcing refresh - validate now - try: - validation = validate_device_for_import( - device, - api=api_for_validation, - include_vc_detection=vc_detection_enabled, - force_vc_refresh=clear_cache, - ) - except (BrokenPipeError, ConnectionError, IOError) as e: - if request: - logger.info(f"Client disconnected during device validation: {e}") - return [] - raise - - # Set VC detection metadata - if not vc_detection_enabled: - validation["virtual_chassis"] = empty_virtual_chassis_data() - - # Apply exclude_existing filter if enabled - if exclude_existing and validation["existing_device"]: - continue - - device["_validation"] = validation - validated_devices.append(device) - - # Cache with TWO keys for different purposes: - # 1. Complex key (with filter context) - for full validated device with all metadata - cache.set(cache_key, device, timeout=api.cache_timeout) - - # 2. Simple key (device ID only) - for quick device data lookup by role/rack updates - # This avoids redundant API calls when user interacts with dropdowns - simple_cache_key = get_import_device_cache_key(device_id, api.server_key) - # Cache just the raw device data (not the full validation result) - # This is what get_validated_device_with_selections() expects - device_data_only = {k: v for k, v in device.items() if k != "_validation"} - cache.set(simple_cache_key, device_data_only, timeout=api.cache_timeout) - - # Store cache metadata (timestamp) for all filter operations - # This enables countdown display regardless of background job vs synchronous execution - # Always store metadata when we have validated devices, even if from_cache - # This ensures metadata is available for countdown display - if validated_devices: - from datetime import datetime, timezone - - cache_metadata_key = get_cache_metadata_key( - server_key=api.server_key, filters=filters, vc_enabled=vc_detection_enabled - ) - - # Check if metadata already exists to preserve original timestamp - # BUT: if clear_cache was requested or data came fresh from LibreNMS, update it - existing_metadata = cache.get(cache_metadata_key) - should_update = clear_cache or not from_cache - - if existing_metadata and not should_update: - # Metadata exists and cache wasn't cleared, keep using it (preserves original cache time) - pass - else: - # No metadata exists, OR cache was cleared, OR fresh data - create/update it now - cache_metadata = { - "cached_at": datetime.now(timezone.utc).isoformat(), - "cache_timeout": api.cache_timeout, - "filters": filters, - "vc_enabled": vc_detection_enabled, - "device_count": len(validated_devices), - } - cache.set(cache_metadata_key, cache_metadata, timeout=api.cache_timeout) - - # Maintain cache index for this server to enable listing active searches - cache_index_key = f"librenms_cache_index_{api.server_key}" - cache_index = cache.get(cache_index_key, []) - # Add this cache key if not already in index - if cache_metadata_key not in cache_index: - cache_index.append(cache_metadata_key) - # Store index with same timeout as the metadata - cache.set(cache_index_key, cache_index, timeout=api.cache_timeout) - - if job: - if exclude_existing: - filtered_count = total - len(validated_devices) - job.logger.info( - f"Validation complete: {len(validated_devices)} devices passed filter, " - f"{filtered_count} filtered out (existing devices excluded)" - ) - else: - job.logger.info(f"Validation complete: {len(validated_devices)} devices ready for import") - else: - logger.info(f"Processed {len(validated_devices)} validated devices") - - if return_cache_status: - return validated_devices, from_cache - return validated_devices diff --git a/netbox_librenms_plugin/import_utils/__init__.py b/netbox_librenms_plugin/import_utils/__init__.py new file mode 100644 index 0000000000..b7a08fef59 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/__init__.py @@ -0,0 +1,52 @@ +""" +Utilities for importing devices from LibreNMS to NetBox. + +This package provides functions for: +- Validating LibreNMS devices for import +- Retrieving filtered LibreNMS devices +- Importing single and multiple devices +- Smart matching of NetBox objects +- Permission checking for import operations +- Virtual chassis detection and creation + +All imports below are intentional re-exports so that existing callers +can continue using ``from netbox_librenms_plugin.import_utils import X``. +The F401 suppressions prevent linters from flagging them as unused. +""" + +from .bulk_import import ( # noqa: F401 + bulk_import_devices, + bulk_import_devices_shared, + process_device_filters, +) +from .cache import ( # noqa: F401 + get_active_cached_searches, + get_cache_metadata_key, + get_import_device_cache_key, + get_validated_device_cache_key, +) +from .device_operations import ( # noqa: F401 + _determine_device_name, + fetch_device_with_cache, + get_librenms_device_by_id, + import_single_device, + validate_device_for_import, +) +from .filters import ( # noqa: F401 + _apply_client_filters, + get_device_count_for_filters, + get_librenms_devices_for_import, +) +from .permissions import check_user_permissions, require_permissions # noqa: F401 +from .virtual_chassis import ( # noqa: F401 + _clone_virtual_chassis_data, + _generate_vc_member_name, + _vc_cache_key, + create_virtual_chassis_with_members, + detect_virtual_chassis_from_inventory, + empty_virtual_chassis_data, + get_virtual_chassis_data, + prefetch_vc_data_for_devices, + update_vc_member_suggested_names, +) +from .vm_operations import bulk_import_vms, create_vm_from_librenms # noqa: F401 diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py new file mode 100644 index 0000000000..8f55e0b6cf --- /dev/null +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -0,0 +1,563 @@ +"""Bulk import orchestration for devices and filter processing.""" + +import logging +from typing import List + +from core.choices import JobStatusChoices +from django.core.cache import cache + +from ..librenms_api import LibreNMSAPI +from .cache import get_cache_metadata_key, get_import_device_cache_key, get_validated_device_cache_key +from .device_operations import import_single_device, validate_device_for_import +from .filters import get_librenms_devices_for_import +from .permissions import require_permissions +from .virtual_chassis import ( + create_virtual_chassis_with_members, + empty_virtual_chassis_data, + prefetch_vc_data_for_devices, +) + +logger = logging.getLogger(__name__) + + +def bulk_import_devices_shared( + device_ids: List[int], + server_key: str = None, + sync_options: dict = None, + manual_mappings_per_device: dict = None, + libre_devices_cache: dict = None, + job=None, + user=None, +) -> dict: + """ + Shared function for importing multiple LibreNMS devices to NetBox. + + Used by both synchronous imports and background jobs. Handles per-device error + collection and optional progress logging when job context is provided. + + Args: + device_ids: List of LibreNMS device IDs to import + server_key: LibreNMS server configuration key + sync_options: Sync options to apply to all devices + manual_mappings_per_device: Dict mapping device_id to manual_mappings dict + Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} + libre_devices_cache: Optional dict mapping device_id to pre-fetched device data + to avoid redundant API calls. Example: {123: {...device_data...}} + job: Optional JobRunner instance for progress logging and cancellation checks + user: User performing the import (for permission checks). If job is provided, + user is extracted from job.job.user if not explicitly passed. + + Returns: + dict: Bulk import result with structure: + { + 'total': int, + 'success': List[dict], # Successfully imported devices + 'failed': List[dict], # Failed imports with errors + 'skipped': List[dict], # Skipped devices (already exist, etc.) + 'virtual_chassis_created': int # Number of VCs created + } + + Raises: + PermissionDenied: If user lacks required permissions + + Example: + >>> # Synchronous usage + >>> result = bulk_import_devices_shared([1, 2, 3, 4, 5], user=request.user) + >>> # Background job usage + >>> result = bulk_import_devices_shared([1, 2, 3], job=self) + """ + # Extract user from job if not explicitly provided + if user is None and job is not None: + user = getattr(job.job, "user", None) + + # Check permissions at start of bulk operation + required_perms = [ + "dcim.add_device", + "dcim.add_interface", + "dcim.add_virtualchassis", + ] + require_permissions(user, required_perms, "import devices") + + total = len(device_ids) + success_list = [] + failed_list = [] + skipped_list = [] + vc_created_count = 0 + processed_vc_domains = set() # Track VCs already created by domain + + # Initialize API client once for all devices to avoid repeated config parsing + api = LibreNMSAPI(server_key=server_key) + + for idx, device_id in enumerate(device_ids, start=1): + # Check for job cancellation every 5 devices + if job and idx % 5 == 0: + # Refresh job from DB to get current status + job.job.refresh_from_db() + job_status = job.job.status + status_value = job_status.value if hasattr(job_status, "value") else job_status + if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): + if job.logger: + job.logger.warning(f"Import job cancelled at device {idx} of {total}") + else: + logger.warning(f"Import cancelled at device {idx} of {total}") + break + # Log progress + if job.logger: + job.logger.info(f"Imported device {idx} of {total}") + + try: + # Use cached device data if available to avoid redundant API calls + if libre_devices_cache and device_id in libre_devices_cache: + libre_device = libre_devices_cache[device_id] + success = True + else: + success, libre_device = api.get_device_info(device_id) + + if not success or not libre_device: + error_msg = f"Failed to retrieve device {device_id} from LibreNMS" + failed_list.append({"device_id": device_id, "error": error_msg}) + if job and job.logger: + job.logger.error(error_msg) + else: + logger.error(error_msg) + continue + + validation = validate_device_for_import(libre_device, api=api) + + # Build manual mappings from validation + any provided overrides + device_mappings = {} + + # Get site and device_type from validation + if validation["site"].get("found") and validation["site"].get("site"): + device_mappings["site_id"] = validation["site"]["site"].id + if validation["device_type"].get("found") and validation["device_type"].get("device_type"): + device_mappings["device_type_id"] = validation["device_type"]["device_type"].id + if validation["platform"].get("found") and validation["platform"].get("platform"): + device_mappings["platform_id"] = validation["platform"]["platform"].id + + # Override with any manual mappings provided for this device + if manual_mappings_per_device and device_id in manual_mappings_per_device: + device_mappings.update(manual_mappings_per_device[device_id]) + + result = import_single_device( + device_id, + server_key=server_key, + sync_options=sync_options, + manual_mappings=device_mappings if device_mappings else None, + libre_device=libre_device, + ) + + if result["success"]: + success_list.append( + { + "device_id": device_id, + "device": result["device"], + "message": result["message"], + } + ) + + # Handle virtual chassis creation for stacks + vc_data = validation.get("virtual_chassis", {}) + if vc_data.get("is_stack", False): + vc_domain = f"librenms-{device_id}" + + # Only create VC if we haven't processed this stack yet + # Add to set BEFORE attempting creation to prevent race condition + if vc_domain not in processed_vc_domains: + processed_vc_domains.add(vc_domain) + try: + vc = create_virtual_chassis_with_members( + result["device"], + vc_data["members"], + libre_device, + ) + vc_created_count += 1 + log_msg = f"Created VC '{vc.name}' during bulk import for device {device_id}" + if job and job.logger: + job.logger.info(log_msg) + else: + logger.info(log_msg) + except Exception as vc_error: + # Remove from set on failure so retry is possible + processed_vc_domains.discard(vc_domain) + warn_msg = f"Failed to create VC for device {device_id}: {vc_error}" + if job and job.logger: + job.logger.warning(warn_msg) + else: + logger.warning(warn_msg) + # Don't fail the import, just log the warning + + elif result.get("device"): # Device exists + skipped_list.append({"device_id": device_id, "reason": result["error"]}) + else: # Failed to import + failed_list.append({"device_id": device_id, "error": result["error"]}) + if job and job.logger: + job.logger.error(f"Failed to import device {device_id}: {result['error']}") + + except Exception as e: + error_msg = f"Unexpected error importing device {device_id}: {str(e)}" + if job and job.logger: + job.logger.error(error_msg, exc_info=True) + else: + logger.exception(f"Unexpected error importing device {device_id}") + failed_list.append({"device_id": device_id, "error": str(e)}) + + return { + "total": total, + "success": success_list, + "failed": failed_list, + "skipped": skipped_list, + "virtual_chassis_created": vc_created_count, + } + + +def bulk_import_devices( + device_ids: List[int], + server_key: str = None, + sync_options: dict = None, + manual_mappings_per_device: dict = None, + libre_devices_cache: dict = None, + user=None, +) -> dict: + """ + Import multiple LibreNMS devices to NetBox (synchronous). + + This is the public API for synchronous imports. For background job usage, + use bulk_import_devices_shared() with a job context. + + Args: + device_ids: List of LibreNMS device IDs to import + server_key: LibreNMS server configuration key + sync_options: Sync options to apply to all devices + manual_mappings_per_device: Dict mapping device_id to manual_mappings dict + Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} + libre_devices_cache: Optional dict mapping device_id to pre-fetched device data + to avoid redundant API calls. Example: {123: {...device_data...}} + user: User performing the import (for permission checks) + + Returns: + dict: Bulk import result with structure: + { + 'total': int, + 'success': List[dict], # Successfully imported devices + 'failed': List[dict], # Failed imports with errors + 'skipped': List[dict], # Skipped devices (already exist, etc.) + 'virtual_chassis_created': int # Number of VCs created + } + + Raises: + PermissionDenied: If user lacks required permissions + """ + return bulk_import_devices_shared( + device_ids=device_ids, + server_key=server_key, + sync_options=sync_options, + manual_mappings_per_device=manual_mappings_per_device, + libre_devices_cache=libre_devices_cache, + job=None, # No job context for synchronous imports + user=user, + ) + + +def _refresh_existing_device(validation: dict) -> None: + """Refresh existing_device from DB to pick up changes made in NetBox since caching.""" + existing = validation.get("existing_device") + if not existing or not hasattr(existing, "pk"): + return + try: + from dcim.models import Device + from virtualization.models import VirtualMachine + + if validation.get("import_as_vm"): + refreshed = VirtualMachine.objects.filter(pk=existing.pk).first() + else: + refreshed = Device.objects.filter(pk=existing.pk).first() + + if refreshed: + validation["existing_device"] = refreshed + if hasattr(refreshed, "role") and refreshed.role: + validation["device_role"] = {"found": True, "role": refreshed.role} + else: + # Device was deleted since caching — recompute readiness + validation["existing_device"] = None + validation["existing_match_type"] = None + validation["can_import"] = True + if validation.get("import_as_vm"): + validation["is_ready"] = bool( + validation.get("site", {}).get("found") and validation.get("device_role", {}).get("found") + ) + else: + validation["is_ready"] = bool( + validation.get("site", {}).get("found") + and validation.get("device_type", {}).get("found") + and validation.get("device_role", {}).get("found") + ) + except Exception as e: + existing_id = getattr(existing, "pk", "unknown") if existing else "none" + logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") + + +def process_device_filters( + api: LibreNMSAPI, + filters: dict, + vc_detection_enabled: bool, + clear_cache: bool, + show_disabled: bool, + exclude_existing: bool = False, + job=None, + request=None, + return_cache_status: bool = False, +) -> List[dict] | tuple[List[dict], bool]: + """ + Process LibreNMS device filters and return validated devices. + + Shared function used by both synchronous view and background job processing. + Fetches devices, optionally pre-warms VC cache, validates each device, and + caches results for HTMX row updates. + + Args: + api: LibreNMS API client instance + filters: Filter dict with location, type, os, hostname, sysname, hardware keys + vc_detection_enabled: Whether to detect virtual chassis + clear_cache: Whether to force cache refresh + show_disabled: Whether to include disabled devices + exclude_existing: Whether to exclude devices that already exist in NetBox + job: Optional JobRunner instance for logging job events + request: Optional Django request for client disconnect detection (synchronous only) + return_cache_status: When True, returns (devices, from_cache) tuple + + Returns: + List[dict]: Validated devices with _validation key, or tuple of (devices, from_cache) + if return_cache_status is True. from_cache=True means data was loaded from existing + cache; from_cache=False means data was just fetched from LibreNMS. + """ + # Fetch devices from LibreNMS + if job: + job.logger.info(f"Fetching devices with filters: {filters}") + else: + logger.info(f"Fetching devices with filters: {filters}") + + # Always get cache status internally, even if not returning it + # We need it to determine if metadata should be updated + libre_devices, from_cache = get_librenms_devices_for_import( + api, + filters=filters, + force_refresh=clear_cache, + return_cache_status=True, + ) + + # Filter out disabled devices if requested + if not show_disabled: + libre_devices = [d for d in libre_devices if d.get("status") == 1] + + if job: + job.logger.info(f"Found {len(libre_devices)} devices to process") + else: + logger.info(f"Found {len(libre_devices)} devices") + + # Pre-warm VC cache if needed + if vc_detection_enabled and libre_devices: + device_ids = [d["device_id"] for d in libre_devices] + if job: + job.logger.info( + f"Pre-fetching virtual chassis data for {len(device_ids)} devices. This may take some time..." + ) + else: + logger.info(f"Pre-fetching VC data for {len(device_ids)} devices") + + try: + prefetch_vc_data_for_devices(api, device_ids, force_refresh=clear_cache) + if job: + job.logger.info("Virtual chassis data pre-fetch completed") + except (BrokenPipeError, ConnectionError, IOError) as e: + if request: + logger.info(f"Client disconnected during VC prefetch: {e}") + return [] + raise + + # Validate each device + validated_devices = [] + total = len(libre_devices) + api_for_validation = api if vc_detection_enabled else None + + if job: + job.logger.info(f"Starting validation of {total} devices") + # Initial check if job was already terminated before we even started + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + + if rq_job.is_failed or rq_job.is_stopped: + job.logger.warning("Job was already stopped before validation started") + return [] + except Exception: + # Fall back to DB check if RQ check fails + job.job.refresh_from_db() + if job.job.status == JobStatusChoices.STATUS_FAILED: + job.logger.warning("Job was stopped before validation started") + return [] + else: + logger.info(f"Validating {total} devices") + + for idx, device in enumerate(libre_devices, 1): + # Check for job termination or client disconnect periodically + if idx % 5 == 0 or idx == 1: # Check more frequently (every 5 devices + first device) + if job: + # Check if job was terminated via stop API + # CRITICAL: Check the RQ job status in Redis, not just the DB model + # NetBox's stop endpoint marks the RQ job as failed in Redis + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + + # Check if RQ job is in a stopped state + if rq_job.is_failed or rq_job.is_stopped: + job.logger.info( + f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." + ) + return [] + except Exception: + # If we can't check RQ status, fall back to DB status check + job.job.refresh_from_db() + if job.job.status == JobStatusChoices.STATUS_FAILED: + job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") + return [] + elif request: + # Check for client disconnect + try: + if hasattr(request, "META") and request.META.get("wsgi.input"): + pass + except (BrokenPipeError, ConnectionError, IOError): + logger.info(f"Client disconnected during validation at device {idx}") + return [] + + # Drop any cached validation/meta keys before recomputing + device.pop("_validation", None) + + # Generate shared cache key for this validated device + device_id = device["device_id"] + cache_key = get_validated_device_cache_key( + server_key=api.server_key, + filters=filters, + device_id=device_id, + vc_enabled=vc_detection_enabled, + ) + + # Check if we already have cached validation for this device + # (only if not forcing refresh) + if not clear_cache: + cached_device = cache.get(cache_key) + if cached_device: + # Use cached validation + device["_validation"] = cached_device["_validation"] + + # Refresh existing_device from DB to avoid stale data + # (user may have changed role, name, etc. in NetBox) + _refresh_existing_device(device["_validation"]) + + # Apply exclude_existing filter if enabled + if exclude_existing: + validation = device["_validation"] + if validation["existing_device"]: + continue + + validated_devices.append(device) + continue + + # Not in cache or forcing refresh - validate now + try: + validation = validate_device_for_import( + device, + api=api_for_validation, + include_vc_detection=vc_detection_enabled, + force_vc_refresh=clear_cache, + ) + except (BrokenPipeError, ConnectionError, IOError) as e: + if request: + logger.info(f"Client disconnected during device validation: {e}") + return [] + raise + + # Set VC detection metadata + if not vc_detection_enabled: + validation["virtual_chassis"] = empty_virtual_chassis_data() + + # Apply exclude_existing filter if enabled + if exclude_existing and validation["existing_device"]: + continue + + device["_validation"] = validation + validated_devices.append(device) + + # Cache with TWO keys for different purposes: + # 1. Complex key (with filter context) - for full validated device with all metadata + cache.set(cache_key, device, timeout=api.cache_timeout) + + # 2. Simple key (device ID only) - for quick device data lookup by role/rack updates + # This avoids redundant API calls when user interacts with dropdowns + simple_cache_key = get_import_device_cache_key(device_id, api.server_key) + # Cache just the raw device data (not the full validation result) + # This is what get_validated_device_with_selections() expects + device_data_only = {k: v for k, v in device.items() if k != "_validation"} + cache.set(simple_cache_key, device_data_only, timeout=api.cache_timeout) + + # Store cache metadata (timestamp) for all filter operations + # This enables countdown display regardless of background job vs synchronous execution + # Always store metadata when we have validated devices, even if from_cache + # This ensures metadata is available for countdown display + if validated_devices: + from datetime import datetime, timezone + + cache_metadata_key = get_cache_metadata_key( + server_key=api.server_key, filters=filters, vc_enabled=vc_detection_enabled + ) + + # Check if metadata already exists to preserve original timestamp + # BUT: if clear_cache was requested or data came fresh from LibreNMS, update it + existing_metadata = cache.get(cache_metadata_key) + should_update = clear_cache or not from_cache + + if existing_metadata and not should_update: + # Metadata exists and cache wasn't cleared, keep using it (preserves original cache time) + pass + else: + # No metadata exists, OR cache was cleared, OR fresh data - create/update it now + cache_metadata = { + "cached_at": datetime.now(timezone.utc).isoformat(), + "cache_timeout": api.cache_timeout, + "filters": filters, + "vc_enabled": vc_detection_enabled, + "device_count": len(validated_devices), + } + cache.set(cache_metadata_key, cache_metadata, timeout=api.cache_timeout) + + # Maintain cache index for this server to enable listing active searches + cache_index_key = f"librenms_cache_index_{api.server_key}" + cache_index = cache.get(cache_index_key, []) + # Add this cache key if not already in index + if cache_metadata_key not in cache_index: + cache_index.append(cache_metadata_key) + # Store index with same timeout as the metadata + cache.set(cache_index_key, cache_index, timeout=api.cache_timeout) + + if job: + if exclude_existing: + filtered_count = total - len(validated_devices) + job.logger.info( + f"Validation complete: {len(validated_devices)} devices passed filter, " + f"{filtered_count} filtered out (existing devices excluded)" + ) + else: + job.logger.info(f"Validation complete: {len(validated_devices)} devices ready for import") + else: + logger.info(f"Processed {len(validated_devices)} validated devices") + + if return_cache_status: + return validated_devices, from_cache + return validated_devices diff --git a/netbox_librenms_plugin/import_utils/cache.py b/netbox_librenms_plugin/import_utils/cache.py new file mode 100644 index 0000000000..716e9dc6a1 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/cache.py @@ -0,0 +1,158 @@ +"""Cache key generation and management for device import operations.""" + +import logging + +from django.core.cache import cache + +logger = logging.getLogger(__name__) + + +def get_cache_metadata_key(server_key: str, filters: dict, vc_enabled: bool) -> str: + """ + Generate a consistent cache metadata key from filter parameters. + + Args: + server_key: LibreNMS server identifier + filters: Filter dictionary + vc_enabled: Whether VC detection is enabled + + Returns: + str: Consistent cache key for metadata + """ + # Sort filter items to ensure consistent key generation + filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v) + return f"librenms_filter_cache_metadata_{server_key}_{filter_parts}_{vc_enabled}" + + +def get_active_cached_searches(server_key: str) -> list[dict]: + """ + Retrieve all active cached searches for a server and enrich with display-friendly values. + + Enriches raw filter IDs with human-readable names by looking up location names + from cached choices and converting type codes to display names. + + Args: + server_key: LibreNMS server identifier + + Returns: + List of dicts containing cache metadata with enriched display_filters + """ + from datetime import datetime, timezone + + cache_index_key = f"librenms_cache_index_{server_key}" + cache_index = cache.get(cache_index_key, []) + + active_searches = [] + valid_cache_keys = [] + + # Get location and type choices for enriching display + location_choices = {} + type_choices = { + "": "All Types", + "network": "Network", + "server": "Server", + "storage": "Storage", + "wireless": "Wireless", + "firewall": "Firewall", + "power": "Power", + "appliance": "Appliance", + "printer": "Printer", + "loadbalancer": "Load Balancer", + "other": "Other", + } + + # Get cached location choices for enrichment + location_cache_key = "librenms_locations_choices" + cached_locations = cache.get(location_cache_key) + if cached_locations: + location_choices = dict(cached_locations) + + for cache_key in cache_index: + metadata = cache.get(cache_key) + if metadata: + # Cache still exists, calculate time remaining + cached_at = datetime.fromisoformat(metadata.get("cached_at")) + cache_timeout = metadata.get("cache_timeout", 300) + now = datetime.now(timezone.utc) + age_seconds = (now - cached_at).total_seconds() + remaining_seconds = max(0, cache_timeout - age_seconds) + + if remaining_seconds > 0: + # Add remaining time and cache key + metadata["remaining_seconds"] = int(remaining_seconds) + metadata["cache_key"] = cache_key + + # Enrich filters with human-readable display values + if "filters" in metadata: + display_filters = metadata["filters"].copy() + # Convert location ID to location name + if "location" in display_filters and display_filters["location"] in location_choices: + display_filters["location"] = location_choices[display_filters["location"]] + # Convert type code to display name + if "type" in display_filters and display_filters["type"] in type_choices: + display_filters["type"] = type_choices[display_filters["type"]] + metadata["display_filters"] = display_filters + else: + # Fallback if filters key missing + metadata["display_filters"] = {} + + active_searches.append(metadata) + valid_cache_keys.append(cache_key) + + # Clean up index if any keys have expired + if len(valid_cache_keys) < len(cache_index): + cache.set(cache_index_key, valid_cache_keys, timeout=3600) + + # Sort by most recent first + active_searches.sort(key=lambda x: x.get("cached_at", ""), reverse=True) + + return active_searches + + +def get_validated_device_cache_key(server_key: str, filters: dict, device_id: int | str, vc_enabled: bool) -> str: + """ + Generate a consistent cache key for validated device data. + + This ensures both synchronous and background job processing use the same + cache keys, avoiding duplicate validation work and cache entries. + + Args: + server_key: LibreNMS server key + filters: Filter dict with location, type, os, hostname, sysname, hardware keys + device_id: LibreNMS device ID + vc_enabled: Whether virtual chassis detection was enabled + + Returns: + str: Cache key for the validated device + + Example: + >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) + >>> key + 'validated_device_default_-1234567890_123_vc' + """ + # Sort filters for consistent hashing + filter_hash = hash(str(sorted(filters.items()))) + vc_part = "vc" if vc_enabled else "novc" + return f"validated_device_{server_key}_{filter_hash}_{device_id}_{vc_part}" + + +def get_import_device_cache_key(device_id: int | str, server_key: str = "default") -> str: + """ + Generate cache key for raw LibreNMS device data. + + This key is used to cache raw device data (without validation metadata) + to avoid redundant API calls when users interact with dropdowns during + the import workflow. + + Args: + device_id: LibreNMS device ID + server_key: LibreNMS server identifier for multi-server setups + + Returns: + str: Cache key for the device data + + Example: + >>> get_import_device_cache_key(123, "production") + 'import_device_data_production_123' + """ + return f"import_device_data_{server_key}_{device_id}" diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py new file mode 100644 index 0000000000..539e133c0b --- /dev/null +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -0,0 +1,859 @@ +"""Device validation, import, and fetch operations.""" + +import logging + +from dcim.models import Device, DeviceRole, DeviceType, Rack, Site +from django.core.cache import cache +from virtualization.models import Cluster +from django.db import transaction +from django.utils import timezone + +from ..librenms_api import LibreNMSAPI +from ..utils import ( + find_matching_platform, + find_matching_site, + match_librenms_hardware_to_device_type, +) +from .cache import get_import_device_cache_key +from .virtual_chassis import empty_virtual_chassis_data, get_virtual_chassis_data + +logger = logging.getLogger(__name__) + + +def _determine_device_name( + libre_device: dict, + use_sysname: bool = True, + strip_domain: bool = False, + device_id: int | str = None, +) -> str: + """ + Determine the device/VM name from LibreNMS data. + + Centralized logic for building device names with consistent handling of: + - sysName vs hostname preference + - Domain stripping (avoiding IP addresses) + - Fallback to device_id when name is missing + + Args: + libre_device: Device data from LibreNMS + use_sysname: If True, prefer sysName; if False, use hostname + strip_domain: If True, strip domain suffix (e.g., '.example.com') + device_id: LibreNMS device ID for fallback name generation + + Returns: + str: The determined device name + + Example: + >>> _determine_device_name({'sysName': 'router.example.com', 'hostname': 'router'}, + ... use_sysname=True, strip_domain=True) + 'router' + """ + # Determine base name based on use_sysname preference + if use_sysname: + name = libre_device.get("sysName") or libre_device.get("hostname") + else: + name = libre_device.get("hostname") or libre_device.get("sysName") + + # Fallback to device_id if no name found + if not name: + if device_id is not None: + name = f"device-{device_id}" + else: + name = libre_device.get("device_id", "unknown") + name = f"device-{name}" + + # Strip domain if requested (but not for IP addresses) + if strip_domain and name and "." in name: + try: + from ipaddress import ip_address + + ip_address(name) + # It's a valid IP address, don't strip + except ValueError: + # Not an IP, safe to strip domain + name = name.split(".")[0] + + return name + + +def validate_device_for_import( + libre_device: dict, + import_as_vm: bool = False, + api: "LibreNMSAPI" = None, + *, + include_vc_detection: bool = True, + force_vc_refresh: bool = False, +) -> dict: + """ + Validate if a LibreNMS device can be imported to NetBox. + + Performs comprehensive validation: + - Checks if device already exists in NetBox + - Validates required prerequisites (Site, DeviceType, DeviceRole for devices) + OR (Cluster for VMs) + - Provides smart matching for missing objects + - Detects virtual chassis/stack configuration (if API provided) + - Returns detailed validation status + + Args: + libre_device: Device data from LibreNMS + import_as_vm: If True, validate for VM import instead of device import + api: Optional LibreNMSAPI instance for virtual chassis detection + include_vc_detection: Skip VC detection when False to speed up bulk operations + force_vc_refresh: When True, bypass cached VC data and re-query LibreNMS + + Returns: + dict: Validation result with structure: + { + 'is_ready': bool, # Can import without user intervention + 'can_import': bool, # Can import (possibly after configuration) + 'import_as_vm': bool, # Whether importing as VM + 'existing_device': Device or VirtualMachine or None, + 'issues': List[str], # Blocking issues + 'warnings': List[str], # Non-blocking warnings + 'site': { # Only for devices + 'found': bool, + 'site': Site or None, + 'match_type': str, # 'exact' or None + 'suggestions': List[Site] # Alternative suggestions + }, + 'device_type': { # Only for devices + 'found': bool, + 'device_type': DeviceType or None, + 'match_type': str, # 'exact' or None + 'suggestions': List[dict] # Device types for user selection + }, + 'device_role': { # Only for devices + 'found': bool, # Always False - requires manual selection + 'role': DeviceRole or None, + 'available_roles': List[DeviceRole] # All roles for user selection + }, + 'cluster': { # Only for VMs + 'found': bool, # Always False - requires manual selection + 'cluster': Cluster or None, + 'available_clusters': List[Cluster] # All clusters for user selection + }, + 'platform': { + 'found': bool, + 'platform': Platform or None, + 'match_type': str # 'exact' or None + } + } + + Example: + >>> validation = validate_device_for_import(libre_device) + >>> if validation['is_ready']: + ... import_single_device(libre_device['device_id']) + """ + result = { + "is_ready": False, + "can_import": False, + "import_as_vm": import_as_vm, + "existing_device": None, + "existing_match_type": None, # Track how existing device was matched + "serial_action": None, # None, "link", "conflict", "update_serial", "hostname_differs" + "serial_confirmed": False, # True when librenms_id match and serial matches + "serial_duplicate": False, # True when incoming serial is already on a different device + "name_matches": False, # True when existing device name matches LibreNMS sysName + "name_sync_available": False, # True when existing device name differs from sysName + "suggested_name": None, # sysName to suggest when name_sync_available is True + "device_type_mismatch": False, # True when existing device's type differs from LibreNMS + "issues": [], + "warnings": [], + "virtual_chassis": empty_virtual_chassis_data(), + "site": { + "found": False, + "site": None, + "match_type": None, + "suggestions": [], + }, + "device_type": { + "found": False, + "device_type": None, + "match_type": None, + "suggestions": [], + }, + "device_role": { + "found": False, + "role": None, + "available_roles": [], + }, + "cluster": { + "found": False, + "cluster": None, + "available_clusters": [], + }, + "platform": {"found": False, "platform": None, "match_type": None}, + "rack": { + "found": False, + "rack": None, + "available_racks": [], + }, + } + + try: + # 1. Check if device/VM already exists in NetBox + # Always check both Devices AND VMs to properly detect existing objects + librenms_id = libre_device.get("device_id") + hostname = libre_device.get("hostname", "") + logger.debug( + f"Checking for existing device/VM: " + f"librenms_id={librenms_id} (type={type(librenms_id).__name__}), " + f"hostname={hostname}" + ) + + from virtualization.models import VirtualMachine + + # Check for existing VM first (by librenms_id custom field) + # Always query with int to match custom field type + try: + existing_vm = VirtualMachine.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() + except (ValueError, TypeError): + # librenms_id is not convertible to int; no match will be found + existing_vm = None + + if existing_vm: + logger.info(f"Found existing VM: {existing_vm.name} (matched by librenms_id={librenms_id})") + result["existing_device"] = existing_vm + result["existing_match_type"] = "librenms_id" + result["import_as_vm"] = True # Force VM mode since VM exists + result["can_import"] = False + + # Check if name matches sysName + # Note: name_sync_available/suggested_name are intentionally not set for VMs + # because UpdateDeviceNameView only supports Device objects; VM name-sync + # would require a separate implementation. + sys_name = libre_device.get("sysName") or "" + if sys_name and existing_vm.name == sys_name: + result["name_matches"] = True + + # Check for existing Device (by librenms_id custom field) + # Always query with int to match custom field type + if not result["existing_device"]: + try: + existing_device = Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() + except (ValueError, TypeError): + # librenms_id is not convertible to int; no match will be found + existing_device = None + + if existing_device: + logger.info(f"Found existing device: {existing_device.name} (matched by librenms_id={librenms_id})") + result["existing_device"] = existing_device + result["existing_match_type"] = "librenms_id" + result["can_import"] = False + + # Check if name matches sysName + sys_name = libre_device.get("sysName") or "" + if sys_name and existing_device.name == sys_name: + result["name_matches"] = True + elif sys_name and existing_device.name != sys_name: + result["name_sync_available"] = True + result["suggested_name"] = sys_name + + # Check for serial drift on the linked device + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-": + if existing_device.serial and existing_device.serial == incoming_serial: + result["serial_confirmed"] = True + elif existing_device.serial and existing_device.serial != incoming_serial: + serial_conflict = ( + Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + ) + if serial_conflict: + result["serial_action"] = "conflict" + result["serial_duplicate"] = True + result["warnings"].append( + f"Serial conflict: incoming serial '{incoming_serial}' is already assigned to " + f"device '{serial_conflict.name}' (ID: {serial_conflict.pk}) in NetBox. " + f"Investigate which device should own this serial before updating." + ) + else: + result["serial_action"] = "update_serial" + result["warnings"].append( + f"Serial number differs (NetBox: '{existing_device.serial}', " + f"LibreNMS: '{incoming_serial}'). Hardware may have been replaced." + ) + + # Only check hostname/serial/IP if not already matched by librenms_id + if not result["existing_device"]: + # Check by hostname/name - Check both VMs and Devices for conflicts + existing_vm = VirtualMachine.objects.filter(name__iexact=hostname).first() + existing_device = Device.objects.filter(name__iexact=hostname).first() + + # If BOTH exist with same hostname, it's ambiguous - don't match either + if existing_vm and existing_device: + logger.warning( + f"Hostname conflict: Both VM '{existing_vm.name}' and Device " + f"'{existing_device.name}' exist with hostname '{hostname}'" + ) + result["warnings"].append( + f"Both a VM and Device exist with hostname '{hostname}' in NetBox. " + f"Cannot determine which to match. Please set the librenms_id custom field on the correct object." + ) + # Don't set existing_device, don't block import - let user proceed as new + # This allows them to import and then resolve the conflict manually + elif existing_vm: + logger.info(f"Found existing VM by hostname: {existing_vm.name}") + result["existing_device"] = existing_vm + result["existing_match_type"] = "hostname" + result["import_as_vm"] = True # Force VM mode since VM exists + result["warnings"].append( + f"VM with same hostname exists in NetBox as '{existing_vm.name}' (not linked to LibreNMS)" + ) + result["can_import"] = False + elif existing_device: + logger.info(f"Found existing device by hostname: {existing_device.name}") + result["existing_device"] = existing_device + result["existing_match_type"] = "hostname" + + # Check for serial conflict on hostname-matched device + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-" and existing_device.serial != incoming_serial: + serial_conflict = ( + Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + ) + if serial_conflict: + result["serial_action"] = "conflict" + result["serial_duplicate"] = True + result["warnings"].append( + f"Serial conflict: incoming serial '{incoming_serial}' is already assigned to " + f"device '{serial_conflict.name}' (ID: {serial_conflict.pk}) in NetBox. " + f"Investigate which device should own this serial before importing." + ) + else: + result["serial_action"] = "update_serial" + result["warnings"].append( + f"Hostname matches but serial differs (NetBox: '{existing_device.serial}', " + f"LibreNMS: '{incoming_serial}'). Hardware may have been replaced." + ) + else: + result["warnings"].append( + f"Device with same hostname exists in NetBox as '{existing_device.name}' (not linked to LibreNMS)" + ) + + result["can_import"] = False + + # Check by serial number (strong physical match - hardware identity) + if not result["existing_device"]: + serial = libre_device.get("serial") or "" + if serial and serial != "-" and not import_as_vm: + existing_by_serial = Device.objects.filter(serial=serial).first() + if existing_by_serial: + logger.info(f"Found existing device by serial: {existing_by_serial.name} (serial={serial})") + result["existing_device"] = existing_by_serial + result["existing_match_type"] = "serial" + result["can_import"] = False + + if existing_by_serial.name and existing_by_serial.name.lower() == hostname.lower(): + result["warnings"].append( + f"Device with same serial and hostname exists as '{existing_by_serial.name}' " + f"(not linked to LibreNMS)" + ) + result["serial_action"] = "link" + else: + result["warnings"].append( + f"Device with same serial ({serial}) exists as '{existing_by_serial.name}' " + f"but hostname differs (LibreNMS: '{hostname}'). Device may have been reinstalled." + ) + result["serial_action"] = "hostname_differs" + + # Check by primary IP (weaker match, IP could be reassigned) - only for devices + if not result["existing_device"]: + primary_ip = libre_device.get("ip") + if primary_ip and not import_as_vm: + from ipam.models import IPAddress + + existing_ip = IPAddress.objects.filter(address__net_host=primary_ip).first() + if existing_ip and existing_ip.assigned_object: + device = ( + existing_ip.assigned_object.device + if hasattr(existing_ip.assigned_object, "device") + else None + ) + if device: + result["existing_device"] = device + result["existing_match_type"] = "primary_ip" + result["warnings"].append( + f"IP address {primary_ip} already assigned to device '{device.name}' (not linked to LibreNMS)" + ) + result["can_import"] = False + + # Validate based on import type (Device or VM) + if import_as_vm: + # 2. For VMs: Validate Cluster (required) - Must be manually selected + result["cluster"]["found"] = False + result["issues"].append("Cluster must be manually selected before importing as VM") + # Provide list of available clusters for user selection (cached) + cache_key = "librenms_import_all_clusters" + all_clusters = cache.get(cache_key) + if all_clusters is None: + all_clusters = list(Cluster.objects.all()) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, all_clusters, cache_timeout) + result["cluster"]["available_clusters"] = all_clusters + + # Skip device-specific validations for VMs + result["site"]["found"] = True # Not required for VMs + result["device_type"]["found"] = True # Not required for VMs + result["device_role"]["found"] = True # Not required for VMs + + else: + # 2. For Devices: Validate Site (required) + location = libre_device.get("location", "") + site_match = find_matching_site(location) + result["site"] = site_match + + if not site_match["found"]: + result["issues"].append(f"No matching site found for location: '{location}'") + # Get alternative suggestions + if location: + all_sites = Site.objects.all()[:10] # Limit for performance + result["site"]["suggestions"] = list(all_sites) + + # 3. Validate DeviceType (required) + hardware = libre_device.get("hardware", "") + dt_match = match_librenms_hardware_to_device_type(hardware) + + result["device_type"] = dt_match + + if not dt_match["matched"]: + result["issues"].append(f"No matching device type found for hardware: '{hardware}'") + # Get some device types for user to choose from + all_device_types = DeviceType.objects.all()[:10] + result["device_type"]["suggestions"] = [ + { + "device_type": dt, + "similarity": 0.0, # No fuzzy matching, just showing options + "match_field": None, + } + for dt in all_device_types + ] + else: + # Rename 'matched' to 'found' for consistency + result["device_type"]["found"] = dt_match["matched"] + result["device_type"]["device_type"] = dt_match["device_type"] + result["device_type"]["match_type"] = dt_match["match_type"] + + # 4. DeviceRole (required) - Must be manually selected by user + logger.debug(f"[{hostname}] Issues BEFORE adding role issue: {result['issues']}") + result["device_role"]["found"] = False + result["issues"].append("Device role must be manually selected before import") + logger.debug(f"[{hostname}] Issues AFTER adding role issue: {result['issues']}") + # Provide list of available roles for user selection (cached) + cache_key = "librenms_import_all_roles" + all_roles = cache.get(cache_key) + if all_roles is None: + all_roles = list(DeviceRole.objects.all()) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, all_roles, cache_timeout) + result["device_role"]["available_roles"] = all_roles + + # 4b. Rack (optional) - Provide available racks for the matched site + if site_match["found"] and site_match["site"]: + site = site_match["site"] + # Use cache to optimize rack lookups per site + cache_key = f"librenms_import_racks_site_{site.pk}" + available_racks = cache.get(cache_key) + + if available_racks is None: + from dcim.models import Rack + from django.db.models import Q + + # Query racks for this site - include both: + # 1. Racks assigned to locations within the site + # 2. Racks directly assigned to the site (without location) + available_racks = list( + Rack.objects.filter(Q(location__site=site) | Q(site=site)) + .select_related("location", "site") + .order_by("location__name", "name") + ) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, available_racks, cache_timeout) + + result["rack"]["available_racks"] = available_racks + # Rack is optional, don't add to issues + result["rack"]["found"] = True # Mark as "found" even if None (optional field) + + # Skip VM-specific validations for devices + result["cluster"]["found"] = True # Not required for devices + + # 5. Match Platform (optional - same for both devices and VMs) + os = libre_device.get("os", "") + platform_match = find_matching_platform(os) + result["platform"] = platform_match + + if not platform_match["found"] and os: + result["warnings"].append(f"No matching platform found for OS: '{os}'") + + # 6. Additional validations + if not hostname: + result["issues"].append("Device has no hostname") + + # 7. Virtual chassis detection (only for devices, not VMs) + if include_vc_detection and not import_as_vm and api is not None: + device_id = libre_device.get("device_id") + if device_id: + try: + logger.debug(f"Calling get_virtual_chassis_data for device {device_id}") + vc_detection = get_virtual_chassis_data(api, device_id, force_refresh=force_vc_refresh) + logger.debug( + f"VC detection result: is_stack={vc_detection.get('is_stack')}, " + f"member_count={vc_detection.get('member_count')}, " + f"members={len(vc_detection.get('members', []))}" + ) + if vc_detection: + result["virtual_chassis"] = vc_detection + if vc_detection["is_stack"]: + logger.debug( + f"Virtual chassis CONFIRMED for device {hostname}: " + f"{vc_detection['member_count']} members" + ) + except Exception as e: + logger.exception(f"Exception during VC detection for device {hostname}: {e}") + result["virtual_chassis"]["detection_error"] = str(e) + else: + logger.debug(f"No device_id found for {hostname}") + + # 8. Determine if device/VM is ready to import + if result["existing_device"]: + # Already matched - can_import was already set to False + result["is_ready"] = False + # Populate role from existing device so the modal shows it + existing = result["existing_device"] + if hasattr(existing, "role") and existing.role: + result["device_role"]["found"] = True + result["device_role"]["role"] = existing.role + + # Check for device type mismatch between existing device and LibreNMS + if hasattr(existing, "device_type") and existing.device_type: + librenms_dt = result["device_type"].get("device_type") + if librenms_dt and existing.device_type.pk != librenms_dt.pk: + result["device_type_mismatch"] = True + result["warnings"].append( + f"Device type mismatch: NetBox has '{existing.device_type}' " + f"but LibreNMS reports '{librenms_dt}'. " + f"This may indicate the wrong device was matched." + ) + else: + result["can_import"] = len(result["issues"]) == 0 + + if import_as_vm: + # For VMs: only cluster is required + result["is_ready"] = result["can_import"] and result["cluster"]["found"] + else: + # For Devices: site, device_type, and device_role are required + result["is_ready"] = ( + result["can_import"] + and result["site"]["found"] + and result["device_type"]["found"] + and result["device_role"]["found"] + ) + + logger.debug( + f"Validation for {libre_device.get('hostname')} ({'VM' if import_as_vm else 'Device'}): " + f"issues={len(result['issues'])}, can_import={result['can_import']}, " + f"issues_list={result['issues']}" + ) + + return result + + except Exception as e: + logger.exception(f"Error validating device for import: {libre_device.get('hostname', 'unknown')}") + result["issues"].append(f"Validation error: {str(e)}") + return result + + +def import_single_device( + device_id: int, + server_key: str = None, + validation: dict = None, + manual_mappings: dict = None, + sync_options: dict = None, + libre_device: dict = None, +) -> dict: + """ + Import a single LibreNMS device to NetBox. + + Args: + device_id: LibreNMS device ID + server_key: LibreNMS server configuration key + validation: Pre-computed validation dict (optional) + manual_mappings: Manual object mappings (optional): + - site_id: NetBox Site ID + - device_type_id: NetBox DeviceType ID + - device_role_id: NetBox DeviceRole ID + - platform_id: NetBox Platform ID (optional) + - rack_id: NetBox Rack ID (optional) + sync_options: Sync options (optional): + - sync_interfaces: bool (default True) + - sync_cables: bool (default True) + - sync_ips: bool (default True) + - sync_fields: bool (default True) + libre_device: Pre-fetched LibreNMS device data (optional). + If provided, skips API call to fetch device info. + + Returns: + dict: Import result with structure: + { + 'success': bool, + 'device': Device object or None, + 'message': str, + 'error': str or None, + 'synced': { + 'interfaces': int, + 'cables': int, + 'ip_addresses': int + } + } + """ + try: + api = LibreNMSAPI(server_key=server_key) + + # Use pre-fetched device data if provided, otherwise fetch from API + if libre_device is None: + success, libre_device = api.get_device_info(device_id) + if not success or not libre_device: + return { + "success": False, + "device": None, + "message": "", + "error": f"Failed to retrieve device {device_id} from LibreNMS", + "synced": {}, + } + + # Validate device if validation not provided + if validation is None: + validation = validate_device_for_import(libre_device) + + # Check if device already exists + if validation.get("existing_device"): + return { + "success": False, + "device": validation["existing_device"], + "message": "", + "error": f"Device already exists: {validation['existing_device'].name}", + "synced": {}, + } + + # Use validation-derived matches, allow manual mappings to override specific fields + site = validation["site"].get("site") + device_type = validation["device_type"].get("device_type") + device_role = validation["device_role"].get("role") + platform = validation["platform"].get("platform") + rack = validation.get("rack", {}).get("rack") + + if manual_mappings: + site = Site.objects.filter(id=manual_mappings.get("site_id")).first() or site + device_type = DeviceType.objects.filter(id=manual_mappings.get("device_type_id")).first() or device_type + device_role = DeviceRole.objects.filter(id=manual_mappings.get("device_role_id")).first() or device_role + + platform_id = manual_mappings.get("platform_id") + if platform_id: + from dcim.models import Platform + + platform = Platform.objects.filter(id=platform_id).first() or platform + + rack_id = manual_mappings.get("rack_id") + if rack_id: + rack = Rack.objects.select_related("location", "site").filter(id=rack_id).first() or rack + + rack = rack or validation.get("rack", {}).get("rack") + + # Validate required fields + if not site: + return { + "success": False, + "device": None, + "message": "", + "error": "Site is required but not provided", + "synced": {}, + } + if not device_type: + return { + "success": False, + "device": None, + "message": "", + "error": "Device type is required but not provided", + "synced": {}, + } + if not device_role: + return { + "success": False, + "device": None, + "message": "", + "error": "Device role is required but not provided", + "synced": {}, + } + + # Create device in NetBox + with transaction.atomic(): + # Determine device name based on sync options + use_sysname = sync_options.get("use_sysname", True) if sync_options else True + strip_domain = sync_options.get("strip_domain", False) if sync_options else False + + device_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=strip_domain, + device_id=device_id, + ) + + # Generate import timestamp comment + import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") + + device_data = { + "name": device_name, + "site": site, + "device_type": device_type, + "role": device_role, + "status": "active" if libre_device.get("status") == 1 else "offline", + "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", + "custom_field_data": {"librenms_id": int(device_id)}, + } + + # Add optional fields + if platform: + device_data["platform"] = platform + + if rack: + device_data["rack"] = rack + + serial = libre_device.get("serial", "") + if serial and serial != "-": + device_data["serial"] = serial + + location_name = libre_device.get("location", "") + if location_name and location_name != "-": + from dcim.models import Location + + # Try to find matching location within the site + location = Location.objects.filter(site=site, name__iexact=location_name).first() + if location: + device_data["location"] = location + + # Create the device + device = Device(**device_data) + device.full_clean() + device.save() + + # Sync additional data based on options + sync_options = sync_options or {} + synced = {"interfaces": 0, "cables": 0, "ip_addresses": 0} + + try: + # Sync interfaces + if sync_options.get("sync_interfaces", True): + # This is simplified - would need proper request context + # For now, just log that it should be done + logger.info(f"Interface sync should be performed for device {device.name}") + + # Sync cables + if sync_options.get("sync_cables", True): + logger.info(f"Cable sync should be performed for device {device.name}") + + # Sync IP addresses + if sync_options.get("sync_ips", True): + logger.info(f"IP address sync should be performed for device {device.name}") + + except Exception as e: + logger.warning(f"Error during post-import sync: {str(e)}") + # Don't fail the import if sync fails + + return { + "success": True, + "device": device, + "message": f"Successfully imported device: {device.name}", + "error": None, + "synced": synced, + } + + except Exception as e: + logger.exception(f"Error importing device {device_id}") + return { + "success": False, + "device": None, + "message": "", + "error": str(e), + "synced": {}, + } + + +def get_librenms_device_by_id(api: LibreNMSAPI, device_id: int) -> dict: + """ + Retrieve a single device from LibreNMS by ID. + + Args: + api: LibreNMSAPI instance + device_id: LibreNMS device ID + + Returns: + Device dictionary or None if not found + """ + try: + # Use the dedicated API endpoint to get device by ID + success, device = api.get_device_info(device_id) + if success and device: + return device + + logger.warning(f"Device {device_id} not found in LibreNMS") + return None + except Exception as e: + logger.exception(f"Failed to get device {device_id} from LibreNMS: {e}") + return None + + +def fetch_device_with_cache( + device_id: int, + api: LibreNMSAPI, + server_key: str = None, + libre_devices_cache: dict = None, +) -> dict | None: + """ + Fetch LibreNMS device from cache or API with automatic caching. + + Checks three sources in order: + 1. Pre-fetched cache dict (if provided) + 2. Django cache (Redis/memory) + 3. LibreNMS API (caches result for future use) + + This function consolidates the device fetching pattern used throughout + the import workflow, eliminating code duplication. + + Args: + device_id: LibreNMS device ID to fetch + api: LibreNMSAPI instance for fallback API calls + server_key: Optional server key for multi-server setups (defaults to api.server_key) + libre_devices_cache: Optional pre-fetched device cache dict + + Returns: + Device dict from LibreNMS, or None if not found + + Example: + >>> # Simple usage + >>> libre_device = fetch_device_with_cache(123, api) + >>> if libre_device: + ... print(libre_device['hostname']) + >>> + >>> # With pre-fetched cache dict + >>> cache_dict = {123: {...}, 456: {...}} + >>> libre_device = fetch_device_with_cache(123, api, libre_devices_cache=cache_dict) + """ + # Check pre-fetched cache dict first (fastest) + if libre_devices_cache and device_id in libre_devices_cache: + return libre_devices_cache[device_id] + + # Check Django cache + cache_key = get_import_device_cache_key(device_id, server_key or api.server_key) + libre_device = cache.get(cache_key) + + if not libre_device: + # Fallback to API fetch + libre_device = get_librenms_device_by_id(api, device_id) + if libre_device: + # Cache for future use + cache.set(cache_key, libre_device, timeout=api.cache_timeout) + + return libre_device diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py new file mode 100644 index 0000000000..27f4449266 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -0,0 +1,253 @@ +"""Device filtering and retrieval from LibreNMS.""" + +import logging +from typing import List + +from django.core.cache import cache + +from ..librenms_api import LibreNMSAPI + +logger = logging.getLogger(__name__) + + +def get_device_count_for_filters( + api: LibreNMSAPI, + filters: dict, + clear_cache: bool = False, + show_disabled: bool = True, +) -> int: + """ + Get count of LibreNMS devices matching filters. + + This is a lightweight function to determine device count for background job + decision making. Uses the same caching as get_librenms_devices_for_import(). + + Args: + api: LibreNMS API client instance + filters: Filter dict with location, type, os, hostname, sysname keys + clear_cache: Whether to force cache refresh + show_disabled: Whether to include disabled devices + + Returns: + int: Count of devices matching filters + """ + devices = get_librenms_devices_for_import(api, filters=filters, force_refresh=clear_cache) + + # Filter out disabled devices if requested + if not show_disabled: + devices = [d for d in devices if d.get("status") == 1] + + return len(devices) + + +def get_librenms_devices_for_import( + api: LibreNMSAPI = None, + filters: dict = None, + server_key: str = None, + *, + force_refresh: bool = False, + return_cache_status: bool = False, +) -> List[dict] | tuple[List[dict], bool]: + """ + Retrieve LibreNMS devices based on filters. + + Args: + api: LibreNMSAPI instance (if not provided, creates one with server_key) + filters: Dict containing filter parameters: + - location: LibreNMS location/site filter + - type: Device type filter + - os: Operating system filter + - hostname: Hostname filter (partial match) + - sysname: System name filter (partial match) + - status: Device status filter (1=up, 0=down) + - disabled: Include disabled devices (0=active only, 1=all) + server_key: Key for specific server configuration (used if api not provided) + force_refresh: When True, bypass the cache and fetch fresh data + return_cache_status: When True, returns (devices, from_cache) tuple + + Returns: + List of device dictionaries from LibreNMS, or tuple of (devices, from_cache) + if return_cache_status is True. from_cache=True means data was loaded from + existing cache; from_cache=False means data was just fetched from LibreNMS. + """ + try: + # Use provided API instance or create a new one + if api is None: + api = LibreNMSAPI(server_key=server_key) + + # Build LibreNMS API filters using the type/query format + # LibreNMS API v0 expects ?type=X&query=Y format, not direct parameters + # NOTE: API only supports ONE type/query pair, so we'll use the most + # specific filter for the API and apply others client-side + api_filters = {} + client_filters = {} # Filters to apply after fetching from API + + if filters: + # Check for status filter first - it has special handling + if filters.get("status") is not None: + # Status filter uses special types that don't need query param + if filters["status"] == 1: + api_filters["type"] = "up" + elif filters["status"] == 0: + api_filters["type"] = "down" + + # Save ALL other filters for client-side filtering when status is used + if filters.get("location"): + client_filters["location"] = filters["location"] + if filters.get("type"): + client_filters["type"] = filters["type"] + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + else: + # Priority order for type/query filters: location > type > os > hostname > sysname + # Note: When sysname is combined with other filters, it's applied client-side for partial matching + # When sysname is alone, it uses API exact match (type=sysName) + # Note: hardware is always applied client-side for partial matching + # Use first available for API, save others for client-side filtering + if filters.get("location"): + api_filters["type"] = "location_id" + api_filters["query"] = filters["location"] + # Save remaining filters for client-side + if filters.get("type"): + client_filters["type"] = filters["type"] + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("type"): + api_filters["type"] = "type" + api_filters["query"] = filters["type"] + # Save remaining filters for client-side + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("os"): + api_filters["type"] = "os" + api_filters["query"] = filters["os"] + # Save remaining filters for client-side + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("hostname"): + api_filters["type"] = "hostname" + api_filters["query"] = filters["hostname"] + # Save sysname and hardware for client-side + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("sysname"): + # sysname-only filter: Use API exact match (type=sysName&query=) + # This is safe - returns empty if no exact match found + api_filters["type"] = "sysName" + api_filters["query"] = filters["sysname"] + # Save hardware for client-side + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("hardware"): + # hardware-only filter: apply client-side for partial matching + client_filters["hardware"] = filters["hardware"] + + # Note: disabled filter isn't directly supported by LibreNMS API + # We'll filter client-side if needed + + # Use caching to avoid repeated API calls + # Include both API and client filters in cache key + cache_key = f"librenms_devices_import_{server_key}_{hash(str(api_filters))}_{hash(str(client_filters))}" + from_cache = False + + if force_refresh: + cache.delete(cache_key) + else: + cached_result = cache.get(cache_key) + if cached_result is not None: + # No need to deepcopy - cached data isn't mutated + devices = cached_result + from_cache = True + if return_cache_status: + return devices, from_cache + return devices + + success, devices = api.list_devices(api_filters if api_filters else None) + + if not success: + logger.error(f"Failed to retrieve devices from LibreNMS: {devices}") + if return_cache_status: + return [], False + return [] + + # Apply client-side filters if any + if client_filters: + devices = _apply_client_filters(devices, client_filters) + + # Cache using configured timeout (default 300s) + # No need to deepcopy - Django's cache backend handles serialization + cache.set(cache_key, devices, timeout=api.cache_timeout) + + if return_cache_status: + return devices, from_cache + return devices + + except Exception: + logger.exception("Error retrieving LibreNMS devices for import") + if return_cache_status: + return [], False + return [] + + +def _apply_client_filters(devices: List[dict], filters: dict) -> List[dict]: + """ + Apply client-side filters to device list. + + Args: + devices: List of device dicts from LibreNMS + filters: Dict of filters to apply (location, type, os, hostname, sysname) + + Returns: + Filtered list of devices + """ + filtered = devices + + if filters.get("location"): + location_id = str(filters["location"]) + filtered = [d for d in filtered if str(d.get("location_id", "")) == location_id] + + if filters.get("type"): + device_type = filters["type"].lower() + filtered = [d for d in filtered if d.get("type", "").lower() == device_type] + + if filters.get("os"): + os_filter = filters["os"].lower() + filtered = [d for d in filtered if os_filter in d.get("os", "").lower()] + + if filters.get("hostname"): + hostname_filter = filters["hostname"].lower() + filtered = [d for d in filtered if hostname_filter in d.get("hostname", "").lower()] + + if filters.get("sysname"): + sysname_filter = filters["sysname"].lower() + filtered = [d for d in filtered if sysname_filter in d.get("sysName", "").lower()] + + if filters.get("hardware"): + hardware_filter = filters["hardware"].lower() + filtered = [d for d in filtered if hardware_filter in (d.get("hardware") or "").lower()] + + return filtered diff --git a/netbox_librenms_plugin/import_utils/permissions.py b/netbox_librenms_plugin/import_utils/permissions.py new file mode 100644 index 0000000000..9e9b4521e0 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/permissions.py @@ -0,0 +1,48 @@ +"""Permission check helpers for device import operations.""" + +import logging + +from django.core.exceptions import PermissionDenied + +logger = logging.getLogger(__name__) + + +def check_user_permissions(user, permissions): + """ + Check if user has all required permissions. + + Args: + user: The user object to check permissions for + permissions: List of permission strings (e.g., ['dcim.add_device', 'dcim.add_interface']) + + Returns: + tuple: (has_all_permissions: bool, missing_permissions: list[str]) + + Raises: + PermissionDenied: If user is None (no user context available) + """ + if user is None: + raise PermissionDenied("No user context available for permission check") + + missing = [perm for perm in permissions if not user.has_perm(perm)] + return (len(missing) == 0, missing) + + +def require_permissions(user, permissions, action_description="perform this action"): + """ + Require user has all permissions, raising PermissionDenied if not. + + Args: + user: The user object to check permissions for + permissions: List of permission strings + action_description: Human-readable description for error message + + Raises: + PermissionDenied: If user lacks any required permission + """ + has_perms, missing = check_user_permissions(user, permissions) + if not has_perms: + missing_str = ", ".join(missing) + raise PermissionDenied( + f"You do not have permission to {action_description}. Missing permissions: {missing_str}" + ) diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py new file mode 100644 index 0000000000..79db61e3fe --- /dev/null +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -0,0 +1,440 @@ +"""Virtual chassis detection, creation, and management.""" + +import logging +from typing import List + +from dcim.models import Device, VirtualChassis +from django.core.cache import cache +from django.db import transaction + +from ..librenms_api import LibreNMSAPI + +logger = logging.getLogger(__name__) + + +def empty_virtual_chassis_data() -> dict: + """Public helper for callers that need a blank VC payload.""" + + return { + "is_stack": False, + "member_count": 0, + "members": [], + "detection_error": None, + } + + +def _clone_virtual_chassis_data(data: dict | None) -> dict: + """Return a defensive copy of cached VC data to avoid shared references.""" + + if not data: + return empty_virtual_chassis_data() + + members = [] + for idx, member in enumerate(data.get("members", [])): + member_copy = member.copy() + raw_position = member_copy.get("position", idx) + try: + member_copy["position"] = int(raw_position) + except (TypeError, ValueError): + member_copy["position"] = idx + members.append(member_copy) + + member_count = data.get("member_count") or len(members) + + return { + "is_stack": bool(data.get("is_stack")), + "member_count": member_count, + "members": members, + "detection_error": data.get("detection_error"), + } + + +_VC_CACHE_VERSION = "v1" + + +def _vc_cache_key(api: LibreNMSAPI, device_id: int | str) -> str: + server_key = getattr(api, "server_key", "default") + return f"librenms_vc_detection_{_VC_CACHE_VERSION}_{server_key}_{device_id}" + + +def get_virtual_chassis_data(api: LibreNMSAPI, device_id: int | str, *, force_refresh: bool = False) -> dict: + """Fetch (and cache) virtual chassis data for a LibreNMS device.""" + + if not api or device_id is None: + return empty_virtual_chassis_data() + + cache_key = _vc_cache_key(api, device_id) + if not force_refresh: + cached = cache.get(cache_key) + if cached is not None: + return _clone_virtual_chassis_data(cached) + + detection_data = detect_virtual_chassis_from_inventory(api, device_id) + if detection_data and "detection_error" not in detection_data: + detection_data["detection_error"] = None + + cache_value = _clone_virtual_chassis_data(detection_data) if detection_data else empty_virtual_chassis_data() + + cache_timeout = getattr(api, "cache_timeout", 300) or 300 + cache.set(cache_key, cache_value, timeout=cache_timeout) + return _clone_virtual_chassis_data(cache_value) + + +def prefetch_vc_data_for_devices(api: LibreNMSAPI, device_ids: List[int], *, force_refresh: bool = False) -> None: + """ + Pre-warm the virtual chassis cache for multiple devices. + + This eliminates the 0.5-1s delay when rendering the import table + by proactively fetching VC data before validation. + + Args: + api: LibreNMSAPI instance + device_ids: List of LibreNMS device IDs to prefetch VC data for + force_refresh: When True, bypass cache and fetch fresh data + + Example: + >>> # Before rendering import table + >>> prefetch_vc_data_for_devices(api, [123, 124, 125]) + >>> # Now all validate_device_for_import() calls hit cache instantly + """ + if not api or not device_ids: + return + + logger.debug(f"Pre-warming VC cache for {len(device_ids)} devices") + + for idx, device_id in enumerate(device_ids): + # This populates the cache if empty, or skips if already cached + try: + get_virtual_chassis_data(api, device_id, force_refresh=force_refresh) + except (BrokenPipeError, ConnectionError, IOError, OSError) as e: + logger.warning(f"Connection error during VC prefetch at device {idx}: {e}") + # Stop processing if connection is broken + return + except Exception as e: + # Log but continue for other errors + logger.warning(f"Error prefetching VC data for device {device_id}: {e}") + + logger.debug(f"VC cache warming complete for {len(device_ids)} devices") + + +def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> dict: + """ + Detect if device is a stack/Virtual Chassis by analyzing ENTITY-MIB inventory. + Vendor-agnostic using standard hierarchical structure. + + Args: + api: LibreNMSAPI instance + device_id: LibreNMS device ID + + Returns: + dict with structure: + { + 'is_stack': bool, + 'member_count': int, + 'members': [ + { + 'serial': str, + 'position': int, + 'model': str, + 'name': str, + 'index': int, + 'description': str, + 'suggested_name': str # Generated using master device name + } + ] + } + Returns None if not a stack or detection fails. + + Detection Logic: + 1. Check root level (entPhysicalContainedIn=0) for parent container + 2. Find parent index (entPhysicalClass='stack' or 'chassis') + 3. Get children chassis at that parent's index + 4. If multiple chassis found -> Stack detected + """ + try: + # Get the master device info to use for naming + success, device_info = api.get_device_info(device_id) + master_name = None + if success and device_info: + master_name = device_info.get("sysName") or device_info.get("hostname") + + # Step 1: Get root level items + success, root_items = api.get_inventory_filtered(device_id, ent_physical_contained_in=0) + + if not success or not root_items: + logger.debug(f"No root inventory items found for device {device_id}") + return None + + # Step 2: Find parent container index + # Could be class="stack" or the main "chassis" + parent_index = None + for item in root_items: + item_class = item.get("entPhysicalClass") + if item_class in ["stack", "chassis"]: + parent_index = item.get("entPhysicalIndex") + logger.debug(f"VC detection: Found parent container at index {parent_index} for device {device_id}") + break + + if not parent_index: + return None + + # Step 3: Get children chassis at next level + success, child_items = api.get_inventory_filtered( + device_id, + ent_physical_class="chassis", + ent_physical_contained_in=parent_index, + ) + + if not success: + return None + + # Filter for chassis only (in case API filter didn't work) + chassis_items = [item for item in (child_items or []) if item.get("entPhysicalClass") == "chassis"] + + # Step 4: Multiple chassis = stack + if len(chassis_items) <= 1: + return None + + # Step 5: Extract member info + members = [] + for idx, chassis in enumerate(chassis_items): + raw_position = chassis.get("entPhysicalParentRelPos", idx) + try: + position = int(raw_position) + except (TypeError, ValueError): + position = idx + member_data = { + "serial": chassis.get("entPhysicalSerialNum", ""), + "position": position, + "model": chassis.get("entPhysicalModelName", ""), + "name": chassis.get("entPhysicalName", ""), + "index": chassis.get("entPhysicalIndex"), + "description": chassis.get("entPhysicalDescr", ""), + } + + # Generate suggested name if we have master name + if master_name: + member_data["suggested_name"] = _generate_vc_member_name(master_name, position + 1) + else: + member_data["suggested_name"] = f"Member-{position + 1}" + + members.append(member_data) + + # Sort by position + members.sort(key=lambda m: m["position"]) + + logger.info(f"Detected stack with {len(members)} members for device {device_id}") + + return {"is_stack": True, "member_count": len(members), "members": members} + + except Exception as e: + logger.exception(f"Error detecting virtual chassis for device {device_id}: {e}") + return None + + +def _generate_vc_member_name(master_name: str, position: int, serial: str = None) -> str: + """ + Generate name for VC member device using configured pattern from settings. + + Args: + master_name: Name of the master/primary device + position: VC position number + serial: Optional serial number of the member device + + Returns: + Generated member device name + + Examples: + pattern="-M{position}" -> "switch01-M2" + pattern=" ({position})" -> "switch01 (2)" + pattern="-SW{position}" -> "switch01-SW2" + pattern=" [{serial}]" -> "switch01 [ABC123]" + """ + # Import here to avoid circular dependency + from ..models import LibreNMSSettings + + # Get pattern from settings with fallback to default + try: + settings = LibreNMSSettings.objects.first() + pattern = settings.vc_member_name_pattern if settings else "-M{position}" + except Exception as e: + logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") + pattern = "-M{position}" + + # Prepare format variables + format_vars = { + "master_name": master_name, + "position": position, + "serial": serial or "", + } + + # Apply pattern - pattern should be suffix/prefix, not full name + try: + formatted_suffix = pattern.format(**format_vars) + return f"{master_name}{formatted_suffix}" + except KeyError as e: + logger.error(f"Invalid placeholder in VC naming pattern '{pattern}': {e}. Using default.") + return f"{master_name}-M{position}" + + +def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: + """ + Regenerate suggested VC member names using the actual master device name. + + This ensures preview shows accurate names after use_sysname and strip_domain + are applied to the master device name. + + Args: + vc_data: Virtual chassis detection data dict + master_name: The actual name that will be used for master device in NetBox + + Returns: + Updated vc_data dict with corrected suggested_name for each member + """ + if not vc_data or not vc_data.get("is_stack"): + return vc_data + + for idx, member in enumerate(vc_data.get("members", [])): + raw_position = member.get("position", idx) + try: + base_position = int(raw_position) + except (TypeError, ValueError): + base_position = idx + position = base_position + 1 # Convert to 1-based position + member["position"] = base_position + member["suggested_name"] = _generate_vc_member_name(master_name, position, serial=member.get("serial")) + + return vc_data + + +def create_virtual_chassis_with_members(master_device: Device, members_info: list, libre_device: dict): + """ + Create Virtual Chassis and member devices from detection info. + + This function creates a NetBox VirtualChassis with the master device + and all detected member devices, wrapped in a transaction for safety. + + Args: + master_device: The imported device (becomes VC master) + members_info: List of member dicts from VC detection + libre_device: Original LibreNMS device data + + Returns: + VirtualChassis: The created virtual chassis instance + + Raises: + ValidationError: If member count validation fails + IntegrityError: If duplicate serials/names are detected + Exception: For other creation errors + + Example members_info: + [ + {'serial': 'ABC123', 'position': 0, 'model': 'C9300-48U', 'name': 'Switch 1'}, + {'serial': 'ABC124', 'position': 1, 'model': 'C9300-48U', 'name': 'Switch 2'} + ] + """ + + # Store original master device state for rollback + original_master_name = master_device.name + original_vc = master_device.virtual_chassis + original_vc_position = master_device.vc_position + + try: + with transaction.atomic(): + # Rename master device to include position 1 pattern + master_device_new_name = _generate_vc_member_name(original_master_name, 1, serial=master_device.serial) + + # Check if renamed master conflicts with existing device + if Device.objects.filter(name=master_device_new_name).exclude(pk=master_device.pk).exists(): + logger.warning( + f"Cannot rename master to '{master_device_new_name}' - name already exists. " + f"Keeping original name '{original_master_name}'" + ) + master_base_name = original_master_name + else: + master_device.name = master_device_new_name + master_base_name = original_master_name + + # Create VC using original base name + vc_name = master_base_name + vc = VirtualChassis.objects.create( + name=vc_name, + master=master_device, + domain=f"librenms-{libre_device['device_id']}", + ) + + # Update master device + master_device.virtual_chassis = vc + master_device.vc_position = 1 # Master is position 1 + master_device.save() + + # Create member devices for remaining positions + position = 2 # Start at 2 (master is 1) + members_created = 0 + + for member in members_info: + # Skip if this is the master's serial + if member.get("serial") == master_device.serial: + continue + + serial = member.get("serial") + + member_rack = master_device.rack + member_location = master_device.location or ( + member_rack.location if member_rack and member_rack.location else None + ) + + # Check for duplicate serial + if serial and Device.objects.filter(serial=serial).exists(): + logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") + continue + + member_name = _generate_vc_member_name(master_base_name, position, serial=serial) + + # Check for duplicate name + if Device.objects.filter(name=member_name).exists(): + logger.warning(f"Device with name '{member_name}' already exists, skipping VC member creation") + continue + + Device.objects.create( + name=member_name, + device_type=master_device.device_type, + role=master_device.role, + site=master_device.site, + location=member_location, + rack=member_rack, + platform=master_device.platform, + serial=serial, + virtual_chassis=vc, + vc_position=position, + comments=f"VC member (LibreNMS: {member.get('name', 'Unknown')})\n" + f"Auto-created from stack inventory", + ) + members_created += 1 + position += 1 + + # Validate member count + expected_members = len([m for m in members_info if m.get("serial") != master_device.serial]) + if members_created < expected_members: + logger.warning( + f"Created {members_created} members but expected {expected_members}. " + "Some members may have been skipped due to duplicates." + ) + + logger.info( + f"Created Virtual Chassis '{vc.name}' with {vc.members.count()} total members " + f"(1 master + {members_created} additional)" + ) + + return vc + + except Exception as e: + # Rollback master device to original state + logger.error( + f"Virtual Chassis creation failed for device {master_device.name}: {e}. Rolling back master device changes." + ) + master_device.name = original_master_name + master_device.virtual_chassis = original_vc + master_device.vc_position = original_vc_position + master_device.save() + raise diff --git a/netbox_librenms_plugin/import_utils/vm_operations.py b/netbox_librenms_plugin/import_utils/vm_operations.py new file mode 100644 index 0000000000..eadd97d200 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/vm_operations.py @@ -0,0 +1,216 @@ +"""Virtual machine creation and import operations.""" + +import logging + +from dcim.models import DeviceRole +from django.utils import timezone +from virtualization.models import Cluster + +from ..librenms_api import LibreNMSAPI +from .device_operations import _determine_device_name, fetch_device_with_cache, validate_device_for_import +from .permissions import require_permissions + +logger = logging.getLogger(__name__) + + +def create_vm_from_librenms(libre_device: dict, validation: dict, use_sysname: bool = True, role=None): + """ + Create a NetBox VirtualMachine from LibreNMS device data. + + Args: + libre_device: Device data from LibreNMS + validation: Validation result from validate_device_for_import with import_as_vm=True + use_sysname: If True, prefer sysName; if False, use hostname + role: Optional DeviceRole to assign to the VM + + Returns: + Created VirtualMachine instance + + Raises: + Exception if VM cannot be created + """ + from virtualization.models import VirtualMachine + + if not validation["can_import"]: + raise ValueError(f"VM cannot be imported: {', '.join(validation['issues'])}") + + # Extract matched objects from validation + cluster = validation["cluster"]["cluster"] + platform = validation["platform"].get("platform") + + # Determine VM name - use pre-computed name if available (handles strip_domain) + vm_name = libre_device.get("_computed_name") + if not vm_name: + vm_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=False, + device_id=libre_device.get("device_id"), + ) + + # Generate import timestamp comment + import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") + + # Create the VM with librenms_id custom field + vm = VirtualMachine.objects.create( + name=vm_name, + cluster=cluster, + role=role, # Optional VM role + platform=platform, + comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", + custom_field_data={"librenms_id": int(libre_device["device_id"])}, + ) + + logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") + return vm + + +def bulk_import_vms( + vm_imports: dict[int, dict[str, int]], + api: LibreNMSAPI, + sync_options: dict = None, + libre_devices_cache: dict = None, + job=None, + user=None, +) -> dict: + """ + Import multiple LibreNMS devices as VMs in NetBox. + + Handles validation, cluster/role assignment, name determination, + and VM creation. Supports both synchronous and background job execution. + + This function consolidates VM import logic that was previously duplicated + in BulkImportDevicesView and ImportDevicesJob, ensuring consistent behavior + across synchronous and background import paths. + + Args: + vm_imports: Dict mapping device_id to {"cluster_id": int, "device_role_id": int} + api: LibreNMSAPI instance for device fetching + sync_options: Optional dict with use_sysname, strip_domain settings + libre_devices_cache: Optional pre-fetched device data cache + job: Optional JobRunner instance for background job logging/cancellation + user: User performing the import (for permission checks). If job is provided, + user is extracted from job.job.user if not explicitly passed. + + Returns: + Dict with keys: + - success: List of {"device_id": int, "device": VM, "message": str} + - failed: List of {"device_id": int, "error": str} + - skipped: List of {"device_id": int, "reason": str} + + Raises: + PermissionDenied: If user lacks required permissions + + Example: + >>> # Synchronous import from view + >>> vm_imports = {123: {"cluster_id": 5, "device_role_id": 2}} + >>> result = bulk_import_vms(vm_imports, api, sync_options, user=request.user) + >>> print(f"Created {len(result['success'])} VMs") + >>> + >>> # Background job import + >>> result = bulk_import_vms(vm_imports, api, sync_options, cache, job=self) + """ + from netbox_librenms_plugin.import_validation_helpers import ( + apply_cluster_to_validation, + apply_role_to_validation, + ) + + # Extract user from job if not explicitly provided + if user is None and job is not None: + user = getattr(job.job, "user", None) + + # Check permissions at start of bulk operation + require_permissions(user, ["virtualization.add_virtualmachine"], "import VMs") + + result = {"success": [], "failed": [], "skipped": []} + vm_ids = list(vm_imports.keys()) + + # Use job logger if available, otherwise standard logger + log = job.logger if job else logger + + for idx, vm_id in enumerate(vm_ids, start=1): + # Check for job cancellation every 5 VMs + if job and idx % 5 == 0: + job.job.refresh_from_db() + job_status = job.job.status + status_value = job_status.value if hasattr(job_status, "value") else job_status + if status_value in ("failed", "errored"): + log.warning(f"Job cancelled at VM {idx} of {len(vm_ids)}") + break + log.info(f"Imported VM {idx} of {len(vm_ids)}") + + try: + # Fetch device data (uses cache helper) + libre_device = fetch_device_with_cache(vm_id, api, api.server_key, libre_devices_cache) + + if not libre_device: + result["failed"].append( + { + "device_id": vm_id, + "error": f"Device {vm_id} not found in LibreNMS", + } + ) + log.error(f"Device {vm_id} not found in LibreNMS") + continue + + # Validate as VM + validation = validate_device_for_import(libre_device, import_as_vm=True, api=api) + + # Check if VM already exists + if validation.get("existing_device"): + result["skipped"].append( + { + "device_id": vm_id, + "reason": f"VM already exists: {validation['existing_device'].name}", + } + ) + log.info(f"VM already exists: {validation['existing_device'].name}") + continue + + # Apply manual cluster and role selections + vm_mappings = vm_imports[vm_id] + cluster_id = vm_mappings.get("cluster_id") + role_id = vm_mappings.get("device_role_id") + + if cluster_id: + cluster = Cluster.objects.filter(id=cluster_id).first() + if cluster: + apply_cluster_to_validation(validation, cluster) + + role = None + if role_id: + role = DeviceRole.objects.filter(id=role_id).first() + if role: + apply_role_to_validation(validation, role, is_vm=True) + + # Determine VM name + use_sysname = sync_options.get("use_sysname", True) if sync_options else True + strip_domain = sync_options.get("strip_domain", False) if sync_options else False + + vm_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=strip_domain, + device_id=vm_id, + ) + + # Update validation with computed name + libre_device["_computed_name"] = vm_name + + # Create VM + vm = create_vm_from_librenms(libre_device, validation, use_sysname=use_sysname, role=role) + + result["success"].append( + { + "device_id": vm_id, + "device": vm, + "message": f"VM {vm.name} created successfully", + } + ) + log.info(f"Successfully imported VM {vm.name} (ID: {vm_id})") + + except Exception as vm_error: + log.error(f"Failed to import VM {vm_id}: {vm_error}", exc_info=True) + result["failed"].append({"device_id": vm_id, "error": str(vm_error)}) + + return result From c603a295d5edf6d0864769e1392308f01661ed3d Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 26 Feb 2026 21:58:44 +0100 Subject: [PATCH 02/27] fix: update mock.patch paths for import_utils package refactor After splitting import_utils.py into a package, mock.patch decorators must target the actual submodule where each name is looked up, not the package __init__.py. Update all patch paths in test_import_utils.py and test_permissions.py to point to the correct submodules. --- .../import_utils/device_operations.py | 2 +- .../tests/test_import_utils.py | 268 +++++++++--------- .../tests/test_permissions.py | 20 +- 3 files changed, 145 insertions(+), 145 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index 539e133c0b..7d10cb959e 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -4,7 +4,7 @@ from dcim.models import Device, DeviceRole, DeviceType, Rack, Site from django.core.cache import cache -from virtualization.models import Cluster +from virtualization.models import Cluster # noqa: F401 — used by test mock.patch targets from django.db import transaction from django.utils import timezone diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 4643106c19..8e6d67c18f 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -149,8 +149,8 @@ def test_determine_device_name_fallback_to_device_id(self): class TestDeviceRetrieval: """Test device retrieval and filtering functions.""" - @patch("netbox_librenms_plugin.import_utils.cache") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.filters.cache") + @patch("netbox_librenms_plugin.import_utils.filters.LibreNMSAPI") def test_get_librenms_devices_for_import_success(self, mock_api_class, mock_cache): """Retrieve devices from LibreNMS API.""" mock_cache.get.return_value = None # Cache miss @@ -171,7 +171,7 @@ def test_get_librenms_devices_for_import_success(self, mock_api_class, mock_cach assert len(devices) == 2 assert devices[0]["hostname"] == "switch-01" - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_librenms_devices_for_import_uses_cache(self, mock_cache): """Cached results returned on repeat call.""" cached_devices = [ @@ -189,7 +189,7 @@ def test_get_librenms_devices_for_import_uses_cache(self, mock_cache): assert devices[0]["hostname"] == "cached-device" mock_api.list_devices.assert_not_called() - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_librenms_devices_for_import_cache_miss(self, mock_cache): """API called when cache empty.""" mock_cache.get.return_value = None @@ -209,7 +209,7 @@ def test_get_librenms_devices_for_import_cache_miss(self, mock_cache): mock_api.list_devices.assert_called_once() assert len(devices) == 1 - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_device_count_for_filters_success(self, mock_cache): """Returns correct count from API.""" mock_cache.get.return_value = [ @@ -225,7 +225,7 @@ def test_get_device_count_for_filters_success(self, mock_cache): assert count == 3 - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_device_count_excludes_disabled(self, mock_cache): """Count respects show_disabled filter parameter.""" mock_cache.get.return_value = [ @@ -279,7 +279,7 @@ def test_empty_virtual_chassis_data(self): assert data["members"] == [] assert data["detection_error"] is None - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.virtual_chassis.cache") def test_get_virtual_chassis_data_returns_empty_without_api(self, mock_cache): """Get VC data returns empty structure without API.""" from netbox_librenms_plugin.import_utils import get_virtual_chassis_data @@ -299,14 +299,14 @@ class TestDeviceValidation: """Test device validation for import.""" @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_site_match_found( self, mock_site_model, @@ -358,14 +358,14 @@ def test_validate_device_site_match_found( assert result["site"]["site"] == mock_site @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_site_not_found( self, mock_site_model, @@ -416,15 +416,15 @@ def test_validate_device_site_not_found( assert any("site" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") - @patch("netbox_librenms_plugin.import_utils.DeviceType") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceType") def test_validate_device_platform_match_found( self, mock_device_type, @@ -478,14 +478,14 @@ def test_validate_device_platform_match_found( assert result["platform"]["platform"] == mock_platform @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_platform_not_found( self, mock_site_model, @@ -535,14 +535,14 @@ def test_validate_device_platform_not_found( assert result["platform"]["found"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_type_match_found( self, mock_site_model, @@ -594,15 +594,15 @@ def test_validate_device_type_match_found( assert result["device_type"]["device_type"] == mock_dt @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") - @patch("netbox_librenms_plugin.import_utils.DeviceType") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceType") def test_validate_device_type_not_found( self, mock_device_type, @@ -655,14 +655,14 @@ def test_validate_device_type_not_found( assert any("device type" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_role_required( self, mock_site_model, @@ -716,14 +716,14 @@ def test_validate_device_role_required( assert any("role" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_handles_empty_location( self, mock_site_model, @@ -775,14 +775,14 @@ def test_validate_device_handles_empty_location( assert result["site"]["found"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_handles_empty_os( self, mock_site_model, @@ -833,15 +833,15 @@ def test_validate_device_handles_empty_os( assert result["platform"]["found"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") - @patch("netbox_librenms_plugin.import_utils.DeviceType") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceType") def test_validate_device_handles_empty_hardware( self, mock_device_type, @@ -894,14 +894,14 @@ def test_validate_device_handles_empty_hardware( assert result["device_type"]["matched"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_duplicate_detection( self, mock_site_model, @@ -933,14 +933,14 @@ def test_validate_device_duplicate_detection( assert result["can_import"] is False @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_returns_complete_state( self, mock_site_model, @@ -999,17 +999,17 @@ def test_validate_device_returns_complete_state( assert "cluster" in result assert "platform" in result - @patch("netbox_librenms_plugin.import_utils.cache") + @patch("netbox_librenms_plugin.import_utils.device_operations.cache") @patch("virtualization.models.Cluster") @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_import_as_vm( self, mock_site_model, @@ -1064,14 +1064,14 @@ def test_validate_device_import_as_vm( assert result["cluster"]["available_clusters"] == mock_clusters @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.Device") - @patch("netbox_librenms_plugin.import_utils.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.Cluster") - @patch("netbox_librenms_plugin.import_utils.Rack") - @patch("netbox_librenms_plugin.import_utils.Site") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") def test_validate_device_existing_vm_blocks_import( self, mock_site_model, @@ -1108,14 +1108,14 @@ class TestSerialNumberMatching: """Test serial number matching in device validation.""" SERIAL_PATCHES = [ - "netbox_librenms_plugin.import_utils.Site", - "netbox_librenms_plugin.import_utils.Rack", - "netbox_librenms_plugin.import_utils.Cluster", - "netbox_librenms_plugin.import_utils.DeviceRole", - "netbox_librenms_plugin.import_utils.match_librenms_hardware_to_device_type", - "netbox_librenms_plugin.import_utils.find_matching_platform", - "netbox_librenms_plugin.import_utils.find_matching_site", - "netbox_librenms_plugin.import_utils.Device", + "netbox_librenms_plugin.import_utils.device_operations.Site", + "netbox_librenms_plugin.import_utils.device_operations.Rack", + "netbox_librenms_plugin.import_utils.device_operations.Cluster", + "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", + "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", + "netbox_librenms_plugin.import_utils.device_operations.Device", "virtualization.models.VirtualMachine", ] @@ -1489,7 +1489,7 @@ def device_filter(**kwargs): self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] - with patch("netbox_librenms_plugin.import_utils.cache") as mock_cache: + with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: mock_cache.get.return_value = None from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1545,7 +1545,7 @@ def device_filter(**kwargs): self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] - with patch("netbox_librenms_plugin.import_utils.cache") as mock_cache: + with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: mock_cache.get.return_value = None from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1591,7 +1591,7 @@ def device_filter(**kwargs): self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] - with patch("netbox_librenms_plugin.import_utils.cache") as mock_cache: + with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: mock_cache.get.return_value = None from netbox_librenms_plugin.import_utils import validate_device_for_import diff --git a/netbox_librenms_plugin/tests/test_permissions.py b/netbox_librenms_plugin/tests/test_permissions.py index d366965ead..50bc2b6d04 100644 --- a/netbox_librenms_plugin/tests/test_permissions.py +++ b/netbox_librenms_plugin/tests/test_permissions.py @@ -633,8 +633,8 @@ class TestView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin): class TestBulkImportPermissions: """Tests for permission checks in bulk import functions.""" - @patch("netbox_librenms_plugin.import_utils.require_permissions") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions") + @patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") def test_bulk_import_devices_checks_permissions(self, mock_api_class, mock_require): """bulk_import_devices_shared calls require_permissions.""" from netbox_librenms_plugin.import_utils import bulk_import_devices_shared @@ -658,8 +658,8 @@ def test_bulk_import_devices_checks_permissions(self, mock_api_class, mock_requi assert "dcim.add_device" in call_args[0][1] assert "dcim.add_interface" in call_args[0][1] - @patch("netbox_librenms_plugin.import_utils.require_permissions") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions") + @patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") def test_bulk_import_devices_extracts_user_from_job(self, mock_api_class, mock_require): """bulk_import_devices_shared extracts user from job if not provided.""" from netbox_librenms_plugin.import_utils import bulk_import_devices_shared @@ -682,7 +682,7 @@ def test_bulk_import_devices_extracts_user_from_job(self, mock_api_class, mock_r call_args = mock_require.call_args assert job_user == call_args[0][0] - @patch("netbox_librenms_plugin.import_utils.require_permissions") + @patch("netbox_librenms_plugin.import_utils.vm_operations.require_permissions") def test_bulk_import_vms_checks_permissions(self, mock_require): """bulk_import_vms calls require_permissions.""" from netbox_librenms_plugin.import_utils import bulk_import_vms @@ -703,7 +703,7 @@ def test_bulk_import_vms_checks_permissions(self, mock_require): assert user == call_args[0][0] assert "virtualization.add_virtualmachine" in call_args[0][1] - @patch("netbox_librenms_plugin.import_utils.require_permissions") + @patch("netbox_librenms_plugin.import_utils.vm_operations.require_permissions") def test_bulk_import_vms_extracts_user_from_job(self, mock_require): """bulk_import_vms extracts user from job if not provided.""" from netbox_librenms_plugin.import_utils import bulk_import_vms @@ -729,7 +729,7 @@ def test_bulk_import_vms_extracts_user_from_job(self, mock_require): class TestBulkImportPermissionDenied: """Tests for permission denied behavior in bulk import.""" - @patch("netbox_librenms_plugin.import_utils.check_user_permissions") + @patch("netbox_librenms_plugin.import_utils.permissions.check_user_permissions") def test_bulk_import_devices_raises_on_missing_permissions(self, mock_check): """bulk_import_devices_shared raises PermissionDenied when permissions missing.""" import pytest @@ -748,7 +748,7 @@ def test_bulk_import_devices_raises_on_missing_permissions(self, mock_check): server_key="default", ) - @patch("netbox_librenms_plugin.import_utils.check_user_permissions") + @patch("netbox_librenms_plugin.import_utils.permissions.check_user_permissions") def test_bulk_import_vms_raises_on_missing_permissions(self, mock_check): """bulk_import_vms raises PermissionDenied when permissions missing.""" import pytest @@ -873,8 +873,8 @@ def test_htmx_rejects_external_referrer(self): class TestBulkImportVCPermission: """Tests that bulk import checks virtualchassis permission.""" - @patch("netbox_librenms_plugin.import_utils.require_permissions") - @patch("netbox_librenms_plugin.import_utils.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions") + @patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") def test_bulk_import_devices_checks_vc_permission(self, mock_api_class, mock_require): """bulk_import_devices_shared includes dcim.add_virtualchassis in required perms.""" from netbox_librenms_plugin.import_utils import bulk_import_devices_shared From 8e65b6d96278a15c48d777a4b6a588037ede6f4a Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 26 Feb 2026 22:10:42 +0100 Subject: [PATCH 03/27] refactor: move Cluster to module-level import in device_operations Keep Cluster as a top-level import so mock.patch targets can resolve it. Remove the redundant inline import inside validate_device_for_import. --- netbox_librenms_plugin/import_utils/device_operations.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index 7d10cb959e..c4fa96f8b5 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -4,9 +4,9 @@ from dcim.models import Device, DeviceRole, DeviceType, Rack, Site from django.core.cache import cache -from virtualization.models import Cluster # noqa: F401 — used by test mock.patch targets from django.db import transaction from django.utils import timezone +from virtualization.models import Cluster # noqa: F401 — used by test mock.patch targets from ..librenms_api import LibreNMSAPI from ..utils import ( @@ -414,7 +414,6 @@ def validate_device_for_import( # 3. Validate DeviceType (required) hardware = libre_device.get("hardware", "") dt_match = match_librenms_hardware_to_device_type(hardware) - result["device_type"] = dt_match if not dt_match["matched"]: From e92eef64a5648589d87dec9f12b026f9d4c71d9e Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 26 Feb 2026 23:17:48 +0100 Subject: [PATCH 04/27] fix: remove stale virtualization.models.Cluster patch in VM import test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that Cluster is a module-level import in device_operations, the test only needs to mock device_operations.Cluster — the extra patch on virtualization.models.Cluster was leftover from the inline-import era and caused the mock to target the wrong object. --- netbox_librenms_plugin/tests/test_import_utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 8e6d67c18f..7d209a2c79 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1000,7 +1000,6 @@ def test_validate_device_returns_complete_state( assert "platform" in result @patch("netbox_librenms_plugin.import_utils.device_operations.cache") - @patch("virtualization.models.Cluster") @patch("virtualization.models.VirtualMachine") @patch("netbox_librenms_plugin.import_utils.device_operations.Device") @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") @@ -1014,14 +1013,13 @@ def test_validate_device_import_as_vm( self, mock_site_model, mock_rack, - mock_cluster_module, + mock_cluster, mock_role, mock_match_type, mock_find_platform, mock_find_site, mock_device, mock_vm, - mock_cluster_local, mock_cache, ): """Import as VM mode uses cluster instead of site/device_type.""" @@ -1045,8 +1043,7 @@ def test_validate_device_import_as_vm( } mock_role.objects.all.return_value = [] mock_clusters = [MagicMock(id=1, name="VMware Cluster")] - # Cluster is imported locally in the VM path, so we need to mock it there - mock_cluster_local.objects.all.return_value = mock_clusters + mock_cluster.objects.all.return_value = mock_clusters mock_cache.get.return_value = None # Force cache miss to trigger Cluster.objects.all() mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] From d25344bdce71251527f06617cabcb5796d78ed31 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Fri, 27 Feb 2026 15:49:20 +0100 Subject: [PATCH 05/27] feat: honour use_sysname and strip_domain in validation path (port from #227) - Add use_sysname/strip_domain params to validate_device_for_import - Store resolved name in validation result as resolved_name - Update import_single_device, bulk_import_devices_shared, bulk_import_vms to extract and pass naming prefs from sync_options - Add _resolve_naming_preferences() helper in actions.py with fallback chain: POST data -> user pref -> LibreNMSSettings -> plugin default - Update DeviceImportHelperMixin and BulkImportConfirmView to pass prefs - Replace _determine_device_name() in BulkImportConfirmView with resolved_name - Add TestDeviceNamingPreferences (5 tests) to test_import_utils.py --- .../import_utils/bulk_import.py | 9 +- .../import_utils/device_operations.py | 21 ++- .../import_utils/vm_operations.py | 10 +- .../tests/test_import_utils.py | 164 ++++++++++++++++++ .../views/imports/actions.py | 51 +++++- 5 files changed, 243 insertions(+), 12 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 8f55e0b6cf..c6ef9090d3 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -122,7 +122,14 @@ def bulk_import_devices_shared( logger.error(error_msg) continue - validation = validate_device_for_import(libre_device, api=api) + use_sysname_opt = sync_options.get("use_sysname", True) if sync_options else True + strip_domain_opt = sync_options.get("strip_domain", False) if sync_options else False + validation = validate_device_for_import( + libre_device, + api=api, + use_sysname=use_sysname_opt, + strip_domain=strip_domain_opt, + ) # Build manual mappings from validation + any provided overrides device_mappings = {} diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index c4fa96f8b5..0ee983a20a 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -83,6 +83,8 @@ def validate_device_for_import( *, include_vc_detection: bool = True, force_vc_refresh: bool = False, + use_sysname: bool = True, + strip_domain: bool = False, ) -> dict: """ Validate if a LibreNMS device can be imported to NetBox. @@ -101,6 +103,8 @@ def validate_device_for_import( api: Optional LibreNMSAPI instance for virtual chassis detection include_vc_detection: Skip VC detection when False to speed up bulk operations force_vc_refresh: When True, bypass cached VC data and re-query LibreNMS + use_sysname: If True, prefer sysName over hostname (matches import behaviour) + strip_domain: If True, strip domain suffix from device name Returns: dict: Validation result with structure: @@ -149,6 +153,7 @@ def validate_device_for_import( "is_ready": False, "can_import": False, "import_as_vm": import_as_vm, + "resolved_name": None, # Final device name after applying user preferences "existing_device": None, "existing_match_type": None, # Track how existing device was matched "serial_action": None, # None, "link", "conflict", "update_serial", "hostname_differs" @@ -195,7 +200,13 @@ def validate_device_for_import( # 1. Check if device/VM already exists in NetBox # Always check both Devices AND VMs to properly detect existing objects librenms_id = libre_device.get("device_id") - hostname = libre_device.get("hostname", "") + hostname = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=strip_domain, + device_id=librenms_id, + ) + result["resolved_name"] = hostname logger.debug( f"Checking for existing device/VM: " f"librenms_id={librenms_id} (type={type(librenms_id).__name__}), " @@ -625,7 +636,13 @@ def import_single_device( # Validate device if validation not provided if validation is None: - validation = validate_device_for_import(libre_device) + use_sysname_opt = sync_options.get("use_sysname", True) if sync_options else True + strip_domain_opt = sync_options.get("strip_domain", False) if sync_options else False + validation = validate_device_for_import( + libre_device, + use_sysname=use_sysname_opt, + strip_domain=strip_domain_opt, + ) # Check if device already exists if validation.get("existing_device"): diff --git a/netbox_librenms_plugin/import_utils/vm_operations.py b/netbox_librenms_plugin/import_utils/vm_operations.py index eadd97d200..737b0d7b2b 100644 --- a/netbox_librenms_plugin/import_utils/vm_operations.py +++ b/netbox_librenms_plugin/import_utils/vm_operations.py @@ -154,7 +154,15 @@ def bulk_import_vms( continue # Validate as VM - validation = validate_device_for_import(libre_device, import_as_vm=True, api=api) + use_sysname_opt = sync_options.get("use_sysname", True) if sync_options else True + strip_domain_opt = sync_options.get("strip_domain", False) if sync_options else False + validation = validate_device_for_import( + libre_device, + import_as_vm=True, + api=api, + use_sysname=use_sysname_opt, + strip_domain=strip_domain_opt, + ) # Check if VM already exists if validation.get("existing_device"): diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 7d209a2c79..9f48fc3fdc 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -2172,3 +2172,167 @@ def test_platform_out_of_sync(self): assert result["platform_synced"] is False assert result["all_synced"] is False + + +class TestDeviceNamingPreferences: + """Test that validation honours use_sysname and strip_domain user preferences.""" + + def _setup_no_existing(self, mocks): + """Configure mocks so no existing device is found.""" + mock_vm = mocks[-1] # VirtualMachine + mock_device = mocks[-2] # Device + mock_find_site = mocks[-3] + mock_find_platform = mocks[-4] + mock_match_type = mocks[-5] + mock_role = mocks[-6] + mock_rack = mocks[-8] + mock_site_model = mocks[-9] + + mock_vm.objects.filter.return_value.first.return_value = None + mock_device.objects.filter.return_value.first.return_value = None + mock_find_site.return_value = { + "found": False, + "site": None, + "match_type": None, + "confidence": 0.0, + } + mock_find_platform.return_value = { + "found": False, + "platform": None, + "match_type": None, + } + mock_match_type.return_value = { + "matched": False, + "device_type": None, + "match_type": None, + } + mock_role.objects.all.return_value = [] + mock_rack.objects.filter.return_value = [] + mock_site_model.objects.all.return_value = [] + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_resolved_name_uses_sysname_by_default(self, *mocks): + """Default use_sysname=True uses sysName for resolved_name.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "10.0.0.1", + "sysName": "core-switch", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + assert result["resolved_name"] == "core-switch" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_resolved_name_uses_hostname_when_sysname_disabled(self, *mocks): + """use_sysname=False uses hostname for resolved_name.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "10.0.0.1", + "sysName": "core-switch", + } + result = validate_device_for_import( + device_data, + include_vc_detection=False, + use_sysname=False, + ) + assert result["resolved_name"] == "10.0.0.1" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_resolved_name_strips_domain(self, *mocks): + """strip_domain=True strips the domain suffix.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "switch-01.example.com", + "sysName": "switch-01.example.com", + } + result = validate_device_for_import( + device_data, + include_vc_detection=False, + strip_domain=True, + ) + assert result["resolved_name"] == "switch-01" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_duplicate_detection_uses_resolved_name(self, *mocks): + """Duplicate detection should match against the resolved name, not raw hostname.""" + self._setup_no_existing(mocks) + + mock_device = mocks[-2] # Device + existing = MagicMock() + existing.name = "core-switch" + existing.serial = "" + mock_device.objects.filter.return_value.first.side_effect = [None, existing] + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 999, + "hostname": "10.0.0.1", + "sysName": "core-switch", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_device"] == existing + assert result["existing_match_type"] == "hostname" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_backward_compatible_defaults(self, *mocks): + """Calling without naming params produces resolved_name in result.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "switch-01", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + assert "resolved_name" in result + assert result["resolved_name"] == "switch-01" diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 5e8a9bc4c5..4bed12e4ae 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -29,12 +29,41 @@ fetch_model_by_id, ) from netbox_librenms_plugin.tables.device_status import DeviceImportTable -from netbox_librenms_plugin.utils import save_user_pref +from netbox_librenms_plugin.utils import get_user_pref, save_user_pref from netbox_librenms_plugin.views.mixins import LibreNMSAPIMixin, LibreNMSPermissionMixin logger = logging.getLogger(__name__) +def _resolve_naming_preferences(request) -> tuple[bool, bool]: + """Resolve use_sysname/strip_domain: POST data → user pref → plugin settings.""" + if "use-sysname-toggle" in request.POST: + use_sysname = request.POST.get("use-sysname-toggle") == "on" + else: + pref = get_user_pref(request, "plugins.netbox_librenms_plugin.use_sysname") + if pref is not None: + use_sysname = pref + else: + from netbox_librenms_plugin.models import LibreNMSSettings + + settings = LibreNMSSettings.objects.first() + use_sysname = getattr(settings, "use_sysname_default", True) if settings else True + + if "strip-domain-toggle" in request.POST: + strip_domain = request.POST.get("strip-domain-toggle") == "on" + else: + pref = get_user_pref(request, "plugins.netbox_librenms_plugin.strip_domain") + if pref is not None: + strip_domain = pref + else: + from netbox_librenms_plugin.models import LibreNMSSettings + + settings = LibreNMSSettings.objects.first() + strip_domain = getattr(settings, "strip_domain_default", False) if settings else False + + return use_sysname, strip_domain + + class DeviceImportHelperMixin: """Mixin providing common validation and rendering helpers for device import views.""" @@ -113,11 +142,16 @@ def get_validated_device_with_selections(self, device_id: int, request) -> tuple # This checks: user preference, cache status, and VM vs Device enable_vc = not is_vm and self._should_enable_vc_detection(device_id, request) + # Extract naming preferences: POST data (hx-include) → user pref → plugin settings. + use_sysname, strip_domain = _resolve_naming_preferences(request) + validation = validate_device_for_import( libre_device, import_as_vm=is_vm, api=self.librenms_api if enable_vc else None, include_vc_detection=enable_vc, + use_sysname=use_sysname, + strip_domain=strip_domain, ) validation["import_as_vm"] = is_vm @@ -259,19 +293,20 @@ def post(self, request): rack_id = selections["rack_id"] is_vm = bool(cluster_id) - validation = validate_device_for_import(libre_device, import_as_vm=is_vm, api=self.librenms_api) + validation = validate_device_for_import( + libre_device, + import_as_vm=is_vm, + api=self.librenms_api, + use_sysname=use_sysname, + strip_domain=strip_domain, + ) # Mark validation with VC detection flag for proper URL generation in table # Bulk confirm should respect the initial filter's VC detection preference vc_requested = request.GET.get("enable_vc_detection") == "true" validation["_vc_detection_enabled"] = vc_requested - device_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=strip_domain, - device_id=device_id, - ) + device_name = validation["resolved_name"] if validation.get("virtual_chassis", {}).get("is_stack") and device_name: validation["virtual_chassis"] = update_vc_member_suggested_names( From d977fe3608d225086277238f7940bcaf7c426abe Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 1 Mar 2026 15:34:46 +0100 Subject: [PATCH 06/27] feat: port serial-matching and conflict-resolution fixes to split import_utils package Port 4 commits from feat/serial-matching-and-conflict-resolution: - 097ea42: Fix validation readiness (VMs require cluster.found), hostname match always sets has_actions, guard platform sync forms for Device only, use _resolve_naming_preferences in BulkImportConfirmView, add else branch for unmatched hardware in _build_sync_info, reuse resolved_name in DeviceConflictActionView actions, fix test patch targets - c0d7bfc: Block import when issues present (can_import/is_ready require no issues), modal title uses resolved_name, dedup LibreNMSSettings DB query in _resolve_naming_preferences, guard hostname form for VMs - 13bafe1: Compare existing device/VM name against resolved_name instead of raw sysName, scope mismatch force gate to link/update/update_serial/ update_type only, add librenms_id collision check before linking - dad0d22: Update device_role dict in-place during refresh to preserve schema keys (available_roles etc.) --- .../import_utils/bulk_import.py | 13 ++-- .../import_utils/device_operations.py | 14 ++-- .../tables/device_status.py | 2 +- .../htmx/device_validation_details.html | 11 ++- .../tests/test_import_utils.py | 30 +++++++- .../views/imports/actions.py | 74 ++++++++++++++----- 6 files changed, 106 insertions(+), 38 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index c6ef9090d3..120c7bda64 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -283,22 +283,25 @@ def _refresh_existing_device(validation: dict) -> None: if refreshed: validation["existing_device"] = refreshed if hasattr(refreshed, "role") and refreshed.role: - validation["device_role"] = {"found": True, "role": refreshed.role} + validation["device_role"]["found"] = True + validation["device_role"]["role"] = refreshed.role else: # Device was deleted since caching — recompute readiness validation["existing_device"] = None validation["existing_match_type"] = None - validation["can_import"] = True if validation.get("import_as_vm"): - validation["is_ready"] = bool( - validation.get("site", {}).get("found") and validation.get("device_role", {}).get("found") + required_found = ( + validation.get("site", {}).get("found") + and validation.get("cluster", {}).get("found") + and validation.get("device_role", {}).get("found") ) else: - validation["is_ready"] = bool( + required_found = ( validation.get("site", {}).get("found") and validation.get("device_type", {}).get("found") and validation.get("device_role", {}).get("found") ) + validation["can_import"] = validation["is_ready"] = bool(required_found and not validation.get("issues")) except Exception as e: existing_id = getattr(existing, "pk", "unknown") if existing else "none" logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index 0ee983a20a..eeb495b6a3 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -230,12 +230,11 @@ def validate_device_for_import( result["import_as_vm"] = True # Force VM mode since VM exists result["can_import"] = False - # Check if name matches sysName + # Check if name matches resolved name (accounts for use_sysname/strip_domain) # Note: name_sync_available/suggested_name are intentionally not set for VMs # because UpdateDeviceNameView only supports Device objects; VM name-sync # would require a separate implementation. - sys_name = libre_device.get("sysName") or "" - if sys_name and existing_vm.name == sys_name: + if hostname and existing_vm.name == hostname: result["name_matches"] = True # Check for existing Device (by librenms_id custom field) @@ -253,13 +252,12 @@ def validate_device_for_import( result["existing_match_type"] = "librenms_id" result["can_import"] = False - # Check if name matches sysName - sys_name = libre_device.get("sysName") or "" - if sys_name and existing_device.name == sys_name: + # Check if name matches resolved name (accounts for use_sysname/strip_domain) + if hostname and existing_device.name == hostname: result["name_matches"] = True - elif sys_name and existing_device.name != sys_name: + elif hostname and existing_device.name != hostname: result["name_sync_available"] = True - result["suggested_name"] = sys_name + result["suggested_name"] = hostname # Check for serial drift on the linked device incoming_serial = libre_device.get("serial") or "" diff --git a/netbox_librenms_plugin/tables/device_status.py b/netbox_librenms_plugin/tables/device_status.py index 71149e58fb..3a548bcd96 100644 --- a/netbox_librenms_plugin/tables/device_status.py +++ b/netbox_librenms_plugin/tables/device_status.py @@ -463,7 +463,7 @@ def render_actions(self, value, record): match_type = validation.get("existing_match_type", "") serial_action = validation.get("serial_action") has_mismatch = validation.get("device_type_mismatch", False) - has_actions = match_type in ("hostname", "serial") and serial_action is not None + has_actions = match_type == "hostname" or (match_type == "serial" and serial_action is not None) has_name_sync = validation.get("name_sync_available", False) has_sync_needed = match_type == "librenms_id" and serial_action in ("update_serial", "conflict") diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html index 974e8b79ec..2bb7fb7f8c 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html @@ -18,7 +18,7 @@
{% else %} {% endif %} - Import Validation: {{ libre_device.sysName|default:libre_device.hostname }} + Import Validation: {{ validation.resolved_name|default:libre_device.sysName|default:libre_device.hostname }}
@@ -256,6 +256,7 @@
Device Information
{{ validation.existing_device.platform }} {% if sync_info and not sync_info.platform_synced %} {% if sync_info.platform_info.platform_exists %} + {% if not validation.import_as_vm and not validation.existing_device.cluster %}
@@ -266,6 +267,7 @@
Device Information
+ {% endif %} {% else %} @@ -278,7 +280,7 @@
Device Information
{{ validation.platform.platform.name }} {% elif sync_info and sync_info.platform_info.platform_exists %} Not set - {% if validation.existing_device %} + {% if validation.existing_device and not validation.import_as_vm and not validation.existing_device.cluster %}
@@ -401,6 +403,11 @@
Device Information
Import blocked: The incoming serial number is already assigned to another device in NetBox. Resolve the duplicate serial before linking. + {% elif validation.import_as_vm %} +
+ + Hostname match found for a VM — use the import action to proceed. +
{% else %}
tuple[bool, bool]: """Resolve use_sysname/strip_domain: POST data → user pref → plugin settings.""" + from netbox_librenms_plugin.models import LibreNMSSettings + + settings = None + if "use-sysname-toggle" in request.POST: use_sysname = request.POST.get("use-sysname-toggle") == "on" else: @@ -44,8 +48,6 @@ def _resolve_naming_preferences(request) -> tuple[bool, bool]: if pref is not None: use_sysname = pref else: - from netbox_librenms_plugin.models import LibreNMSSettings - settings = LibreNMSSettings.objects.first() use_sysname = getattr(settings, "use_sysname_default", True) if settings else True @@ -56,9 +58,8 @@ def _resolve_naming_preferences(request) -> tuple[bool, bool]: if pref is not None: strip_domain = pref else: - from netbox_librenms_plugin.models import LibreNMSSettings - - settings = LibreNMSSettings.objects.first() + if settings is None: + settings = LibreNMSSettings.objects.first() strip_domain = getattr(settings, "strip_domain_default", False) if settings else False return use_sysname, strip_domain @@ -257,8 +258,7 @@ def post(self, request): status=400, ) - use_sysname = request.POST.get("use-sysname-toggle") == "on" - strip_domain = request.POST.get("strip-domain-toggle") == "on" + use_sysname, strip_domain = _resolve_naming_preferences(request) devices = [] errors = [] @@ -795,6 +795,8 @@ def _build_sync_info(libre_device, existing_device): librenms_device_type = hw_match["device_type"] if not existing_device.device_type or existing_device.device_type.pk != librenms_device_type.pk: device_type_synced = False + else: + device_type_synced = False all_synced = serial_synced and platform_synced and device_type_synced @@ -874,9 +876,10 @@ def post(self, request, device_id): if not libre_device: return HttpResponse("LibreNMS device not found", status=404) - # Require force flag when device type mismatches + # Require force flag when device type mismatches, but only for actions that use it + _FORCE_REQUIRED_ACTIONS = {"link", "update", "update_serial", "update_type"} force = request.POST.get("force") == "on" - if validation.get("device_type_mismatch") and not force: + if validation.get("device_type_mismatch") and action in _FORCE_REQUIRED_ACTIONS and not force: return HttpResponse( "Device type mismatch detected. Check the force checkbox to proceed.", status=400, @@ -889,11 +892,32 @@ def post(self, request, device_id): librenms_id = libre_device.get("device_id") + # Check for LibreNMS ID collision before any linking action + if action in {"link", "update", "update_serial"}: + id_conflict = ( + Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)) + .exclude(pk=existing_device.pk) + .first() + ) + if id_conflict: + return HttpResponse( + f"LibreNMS ID conflict: ID {librenms_id} is already assigned to device " + f"'{id_conflict.name}' (ID: {id_conflict.pk})", + status=409, + ) + if action == "link": # Link to LibreNMS and update name from LibreNMS data - use_sysname = request.POST.get("use-sysname-toggle") == "on" - strip_domain = request.POST.get("strip-domain-toggle") == "on" - hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) + resolved_name = validation.get("resolved_name") + hostname = ( + resolved_name + if resolved_name + else _determine_device_name( + libre_device, + use_sysname=request.POST.get("use-sysname-toggle") == "on", + strip_domain=request.POST.get("strip-domain-toggle") == "on", + ) + ) existing_device.custom_field_data["librenms_id"] = int(librenms_id) existing_device.name = hostname if librenms_device_type: @@ -903,10 +927,17 @@ def post(self, request, device_id): elif action == "update": # Update hostname, serial, and link to LibreNMS - use_sysname = request.POST.get("use-sysname-toggle") == "on" - strip_domain = request.POST.get("strip-domain-toggle") == "on" + resolved_name = validation.get("resolved_name") incoming_serial = libre_device.get("serial") or "" - hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) + hostname = ( + resolved_name + if resolved_name + else _determine_device_name( + libre_device, + use_sysname=request.POST.get("use-sysname-toggle") == "on", + strip_domain=request.POST.get("strip-domain-toggle") == "on", + ) + ) existing_device.custom_field_data["librenms_id"] = int(librenms_id) if incoming_serial and incoming_serial != "-": conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() @@ -949,9 +980,16 @@ def post(self, request, device_id): elif action == "sync_name": # Sync device name from LibreNMS (e.g., IP → sysName) - use_sysname = request.POST.get("use-sysname-toggle") == "on" - strip_domain = request.POST.get("strip-domain-toggle") == "on" - hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) + resolved_name = validation.get("resolved_name") + hostname = ( + resolved_name + if resolved_name + else _determine_device_name( + libre_device, + use_sysname=request.POST.get("use-sysname-toggle") == "on", + strip_domain=request.POST.get("strip-domain-toggle") == "on", + ) + ) existing_device.name = hostname existing_device.save() logger.info(f"Synced name on device '{existing_device.name}' from LibreNMS") From 46d6b084de4c3289cd2ade2c47fe9a34d7c687d2 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 1 Mar 2026 15:43:57 +0100 Subject: [PATCH 07/27] fix: add librenms_id collision mock to force-with-mismatch tests The librenms_id conflict check added in the port requires Device.objects.filter().exclude().first() to be mocked for all tests using link/update/update_serial actions. --- netbox_librenms_plugin/tests/test_import_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 4310df3373..96c2016037 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1906,6 +1906,7 @@ def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cach patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1950,6 +1951,7 @@ def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cach patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() From 7408b7ad45a2eb9d4e7a7cb4c3e5717cffe74c7a Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sat, 28 Feb 2026 22:09:18 +0100 Subject: [PATCH 08/27] feat: librenms_id refactor from int to JSON --- .../import_utils/bulk_import.py | 2 + .../import_utils/device_operations.py | 14 +++-- .../import_utils/vm_operations.py | 11 +++- netbox_librenms_plugin/librenms_api.py | 8 ++- netbox_librenms_plugin/tables/interfaces.py | 6 +- .../tests/test_import_utils.py | 16 ++--- netbox_librenms_plugin/utils.py | 60 +++++++++++++++++++ .../views/base/cables_view.py | 20 +++++-- .../views/base/ip_addresses_view.py | 7 ++- .../views/imports/actions.py | 11 ++-- .../views/object_sync/devices.py | 13 +++- .../views/object_sync/vms.py | 4 +- .../views/sync/interfaces.py | 4 +- 13 files changed, 138 insertions(+), 38 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 120c7bda64..0409b35da3 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -129,6 +129,7 @@ def bulk_import_devices_shared( api=api, use_sysname=use_sysname_opt, strip_domain=strip_domain_opt, + server_key=api.server_key, ) # Build manual mappings from validation + any provided overrides @@ -487,6 +488,7 @@ def process_device_filters( api=api_for_validation, include_vc_detection=vc_detection_enabled, force_vc_refresh=clear_cache, + server_key=api.server_key, ) except (BrokenPipeError, ConnectionError, IOError) as e: if request: diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index eeb495b6a3..c21d0bfb7a 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -85,6 +85,7 @@ def validate_device_for_import( force_vc_refresh: bool = False, use_sysname: bool = True, strip_domain: bool = False, + server_key: str = "default", ) -> dict: """ Validate if a LibreNMS device can be imported to NetBox. @@ -216,9 +217,10 @@ def validate_device_for_import( from virtualization.models import VirtualMachine # Check for existing VM first (by librenms_id custom field) - # Always query with int to match custom field type try: - existing_vm = VirtualMachine.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() + from netbox_librenms_plugin.utils import find_by_librenms_id + + existing_vm = find_by_librenms_id(VirtualMachine, int(librenms_id), server_key) except (ValueError, TypeError): # librenms_id is not convertible to int; no match will be found existing_vm = None @@ -238,10 +240,11 @@ def validate_device_for_import( result["name_matches"] = True # Check for existing Device (by librenms_id custom field) - # Always query with int to match custom field type if not result["existing_device"]: try: - existing_device = Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() + from netbox_librenms_plugin.utils import find_by_librenms_id + + existing_device = find_by_librenms_id(Device, int(librenms_id), server_key) except (ValueError, TypeError): # librenms_id is not convertible to int; no match will be found existing_device = None @@ -640,6 +643,7 @@ def import_single_device( libre_device, use_sysname=use_sysname_opt, strip_domain=strip_domain_opt, + server_key=server_key or "default", ) # Check if device already exists @@ -725,7 +729,7 @@ def import_single_device( "role": device_role, "status": "active" if libre_device.get("status") == 1 else "offline", "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - "custom_field_data": {"librenms_id": int(device_id)}, + "custom_field_data": {"librenms_id": {(server_key or "default"): int(device_id)}}, } # Add optional fields diff --git a/netbox_librenms_plugin/import_utils/vm_operations.py b/netbox_librenms_plugin/import_utils/vm_operations.py index 737b0d7b2b..1272871a5a 100644 --- a/netbox_librenms_plugin/import_utils/vm_operations.py +++ b/netbox_librenms_plugin/import_utils/vm_operations.py @@ -13,7 +13,9 @@ logger = logging.getLogger(__name__) -def create_vm_from_librenms(libre_device: dict, validation: dict, use_sysname: bool = True, role=None): +def create_vm_from_librenms( + libre_device: dict, validation: dict, use_sysname: bool = True, role=None, server_key: str = "default" +): """ Create a NetBox VirtualMachine from LibreNMS device data. @@ -58,7 +60,7 @@ def create_vm_from_librenms(libre_device: dict, validation: dict, use_sysname: b role=role, # Optional VM role platform=platform, comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - custom_field_data={"librenms_id": int(libre_device["device_id"])}, + custom_field_data={"librenms_id": {server_key: int(libre_device["device_id"])}}, ) logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") @@ -162,6 +164,7 @@ def bulk_import_vms( api=api, use_sysname=use_sysname_opt, strip_domain=strip_domain_opt, + server_key=api.server_key, ) # Check if VM already exists @@ -206,7 +209,9 @@ def bulk_import_vms( libre_device["_computed_name"] = vm_name # Create VM - vm = create_vm_from_librenms(libre_device, validation, use_sysname=use_sysname, role=role) + vm = create_vm_from_librenms( + libre_device, validation, use_sysname=use_sysname, role=role, server_key=api.server_key + ) result["success"].append( { diff --git a/netbox_librenms_plugin/librenms_api.py b/netbox_librenms_plugin/librenms_api.py index 5de9db6c25..8e2b2b3ac0 100644 --- a/netbox_librenms_plugin/librenms_api.py +++ b/netbox_librenms_plugin/librenms_api.py @@ -190,7 +190,9 @@ def get_librenms_id(self, obj): If found via API, stores ID in custom field if available, otherwise caches the value. """ - librenms_id = obj.cf.get("librenms_id") + from netbox_librenms_plugin.utils import get_librenms_device_id + + librenms_id = get_librenms_device_id(obj, self.server_key) if librenms_id: return librenms_id @@ -254,7 +256,9 @@ def _store_librenms_id(self, obj, librenms_id): None """ if "librenms_id" in obj.cf: - obj.custom_field_data["librenms_id"] = librenms_id + from netbox_librenms_plugin.utils import set_librenms_device_id + + set_librenms_device_id(obj, librenms_id, self.server_key) obj.save() else: # Use cache as fallback diff --git a/netbox_librenms_plugin/tables/interfaces.py b/netbox_librenms_plugin/tables/interfaces.py index 5ceb30bf51..1c5d62b1ec 100644 --- a/netbox_librenms_plugin/tables/interfaces.py +++ b/netbox_librenms_plugin/tables/interfaces.py @@ -13,6 +13,7 @@ convert_speed_to_kbps, format_mac_address, get_interface_name_field, + get_librenms_device_id, get_missing_vlan_warning, get_table_paginate_count, get_tagged_vlan_css_class, @@ -46,11 +47,12 @@ class Meta: "id": "librenms-interface-table", } - def __init__(self, *args, device=None, interface_name_field=None, vlan_groups=None, **kwargs): + def __init__(self, *args, device=None, interface_name_field=None, vlan_groups=None, server_key="default", **kwargs): """Initialize table with device context and interface name field.""" self.device = device self.interface_name_field = interface_name_field or get_interface_name_field() self.vlan_groups = vlan_groups or [] + self.server_key = server_key # Update column accessors after initialization for column in ["selection", "name"]: @@ -360,7 +362,7 @@ def render_librenms_id(self, value, record): if not netbox_interface: return mark_safe(f'{value}') - netbox_librenms_id = netbox_interface.custom_field_data.get("librenms_id") + netbox_librenms_id = get_librenms_device_id(netbox_interface, self.server_key) if netbox_librenms_id is None: return mark_safe( diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 96c2016037..a537f95d1c 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1346,7 +1346,7 @@ def test_librenms_id_match_shows_serial_confirmed(self): def device_filter(**kwargs): result = MagicMock() - if "custom_field_data__librenms_id" in kwargs: + if any(k.startswith("custom_field_data__librenms_id") for k in kwargs): result.first.return_value = existing else: result.first.return_value = None @@ -1386,7 +1386,7 @@ def test_librenms_id_match_detects_serial_drift(self): def device_filter(**kwargs): result = MagicMock() - if "custom_field_data__librenms_id" in kwargs: + if any(k.startswith("custom_field_data__librenms_id") for k in kwargs): result.first.return_value = existing elif "serial" in kwargs: result.first.return_value = None @@ -1428,7 +1428,7 @@ def test_librenms_id_match_still_validates_site(self): def device_filter(**kwargs): result = MagicMock() - if "custom_field_data__librenms_id" in kwargs: + if any(k.startswith("custom_field_data__librenms_id") for k in kwargs): result.first.return_value = existing else: result.first.return_value = None @@ -1665,7 +1665,7 @@ def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache) view.post(request, device_id=10) - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} assert existing_device.name == "switch-01.example.com" existing_device.save.assert_called_once() @@ -1705,7 +1705,7 @@ def test_update_action_sets_hostname_serial_and_librenms_id(self, mock_cache_key view.post(request, device_id=10) - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} assert existing_device.serial == "NEW-SERIAL" assert existing_device.name == "new-name.example.com" existing_device.save.assert_called_once() @@ -1741,7 +1741,7 @@ def test_update_serial_action_updates_serial_only(self, mock_cache_key, mock_cac view.post(request, device_id=10) - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} assert existing_device.serial == "NEW-SERIAL" # Name should NOT be changed by update_serial assert existing_device.name == "switch-01" @@ -1912,7 +1912,7 @@ def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cach view.post(request, device_id=10) - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} existing_device.save.assert_called_once() @patch("netbox_librenms_plugin.views.imports.actions.cache") @@ -1958,7 +1958,7 @@ def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cach view.post(request, device_id=10) assert existing_device.device_type == librenms_device_type - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} existing_device.save.assert_called_once() @patch("netbox_librenms_plugin.views.imports.actions.cache") diff --git a/netbox_librenms_plugin/utils.py b/netbox_librenms_plugin/utils.py index 4a5bf113a4..aae146b187 100644 --- a/netbox_librenms_plugin/utils.py +++ b/netbox_librenms_plugin/utils.py @@ -9,6 +9,66 @@ from utilities.paginator import get_paginate_count as netbox_get_paginate_count +def get_librenms_device_id(obj, server_key: str = "default"): + """ + Get the LibreNMS device/port ID for a specific server from the JSON custom field. + + Supports both the legacy integer format and the new multi-server JSON format:: + + Legacy: librenms_id = 42 → returns 42 for any server_key + New: librenms_id = {"primary": 42} → returns 42 only for server_key="primary" + + Args: + obj: NetBox object with a ``librenms_id`` custom field. + server_key: LibreNMS server key (from plugin ``servers`` config). + + Returns: + int or None + """ + cf_value = obj.cf.get("librenms_id") + if cf_value is None: + return None + if isinstance(cf_value, int): + return cf_value # backward compat: bare integer from pre-migration + if isinstance(cf_value, dict): + return cf_value.get(server_key) + return None + + +def set_librenms_device_id(obj, device_id, server_key: str = "default"): + """ + Set the LibreNMS device/port ID for a specific server on the JSON custom field. + + Migrates any legacy bare-integer value to the dict format on first write. + + Args: + obj: NetBox object with a ``librenms_id`` custom field. + device_id: LibreNMS device ID (integer). + server_key: LibreNMS server key (from plugin ``servers`` config). + """ + cf_value = obj.custom_field_data.get("librenms_id") or {} + if isinstance(cf_value, int): + cf_value = {"default": cf_value} # migrate legacy value on first write + cf_value[server_key] = device_id + obj.custom_field_data["librenms_id"] = cf_value + + +def find_by_librenms_id(model, librenms_id, server_key: str = "default"): + """ + Return the first object of *model* whose ``librenms_id`` JSON field contains + *librenms_id* under *server_key*. + + Args: + model: A Django model class (Device, VirtualMachine, Interface, …). + librenms_id: The LibreNMS device/port ID to look up. + server_key: LibreNMS server key (from plugin ``servers`` config). + + Returns: + Model instance or None + """ + return model.objects.filter(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}).first() + + def convert_speed_to_kbps(speed_bps: int) -> int: """ Convert speed from bits per second to kilobits per second. diff --git a/netbox_librenms_plugin/views/base/cables_view.py b/netbox_librenms_plugin/views/base/cables_view.py index c390cdd539..89c4a72a0e 100644 --- a/netbox_librenms_plugin/views/base/cables_view.py +++ b/netbox_librenms_plugin/views/base/cables_view.py @@ -78,10 +78,11 @@ def get_links_data(self, obj): def get_device_by_id_or_name(self, remote_device_id, hostname): """Try to find device in NetBox first by librenms_id custom field, then by name""" + server_key = self.librenms_api.server_key # First try matching by LibreNMS ID if remote_device_id: try: - device = Device.objects.get(custom_field_data__librenms_id=remote_device_id) + device = Device.objects.get(**{f"custom_field_data__librenms_id__{server_key}": remote_device_id}) return device, True, None except Device.DoesNotExist: pass @@ -116,13 +117,16 @@ def enrich_local_port(self, link, obj): if local_port := link.get("local_port"): interface = None local_port_id = link.get("local_port_id") + server_key = self.librenms_api.server_key if hasattr(obj, "virtual_chassis") and obj.virtual_chassis: chassis_member = get_virtual_chassis_member(obj, local_port) # First try to find interface by librenms_id if local_port_id: - interface = chassis_member.interfaces.filter(custom_field_data__librenms_id=local_port_id).first() + interface = chassis_member.interfaces.filter( + **{f"custom_field_data__librenms_id__{server_key}": local_port_id} + ).first() # Only if librenms_id match fails, try matching by name if not interface: @@ -130,7 +134,9 @@ def enrich_local_port(self, link, obj): else: # First try to find interface by librenms_id if local_port_id: - interface = obj.interfaces.filter(custom_field_data__librenms_id=local_port_id).first() + interface = obj.interfaces.filter( + **{f"custom_field_data__librenms_id__{server_key}": local_port_id} + ).first() # Only if librenms_id match fails, try matching by name if not interface: @@ -145,6 +151,7 @@ def enrich_remote_port(self, link, device): if remote_port := link.get("remote_port"): netbox_remote_interface = None librenms_remote_port_id = link.get("remote_port_id") + server_key = self.librenms_api.server_key # Handle virtual chassis case if hasattr(device, "virtual_chassis") and device.virtual_chassis: @@ -154,7 +161,7 @@ def enrich_remote_port(self, link, device): # First try to find interface by librenms_id if librenms_remote_port_id: netbox_remote_interface = chassis_member.interfaces.filter( - custom_field_data__librenms_id=librenms_remote_port_id + **{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id} ).first() # If not found by librenms_id, fall back to name matching on the correct chassis member @@ -165,7 +172,7 @@ def enrich_remote_port(self, link, device): # First try to find interface by librenms_id if librenms_remote_port_id: netbox_remote_interface = device.interfaces.filter( - custom_field_data__librenms_id=librenms_remote_port_id + **{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id} ).first() # If not found by librenms_id, fall back to name matching @@ -369,8 +376,9 @@ def post(self, request): # First try to find interface by librenms_id interface = None if local_port_id := link_data.get("local_port_id"): + _sk = self.librenms_api.server_key interface = selected_device.interfaces.filter( - custom_field_data__librenms_id=local_port_id + **{f"custom_field_data__librenms_id__{_sk}": local_port_id} ).first() # If not found by librenms_id, try matching by name diff --git a/netbox_librenms_plugin/views/base/ip_addresses_view.py b/netbox_librenms_plugin/views/base/ip_addresses_view.py index 22f4b49742..1ca3f6e290 100644 --- a/netbox_librenms_plugin/views/base/ip_addresses_view.py +++ b/netbox_librenms_plugin/views/base/ip_addresses_view.py @@ -11,7 +11,7 @@ from virtualization.models import VirtualMachine from netbox_librenms_plugin.tables.ipaddresses import IPAddressTable -from netbox_librenms_plugin.utils import get_interface_name_field +from netbox_librenms_plugin.utils import get_interface_name_field, get_librenms_device_id from netbox_librenms_plugin.views.mixins import CacheMixin, LibreNMSAPIMixin, LibreNMSPermissionMixin @@ -104,10 +104,11 @@ def _prefetch_netbox_data(self, obj): all_interfaces = list(obj.interfaces.all()) # Create maps for efficient lookups + server_key = self.librenms_api.server_key interfaces_by_librenms_id = { - interface.custom_field_data.get("librenms_id"): interface + get_librenms_device_id(interface, server_key): interface for interface in all_interfaces - if interface.custom_field_data.get("librenms_id") + if get_librenms_device_id(interface, server_key) } interfaces_by_name = {interface.name: interface for interface in all_interfaces} diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 696a3cc2bd..b0e262a80a 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -29,7 +29,7 @@ fetch_model_by_id, ) from netbox_librenms_plugin.tables.device_status import DeviceImportTable -from netbox_librenms_plugin.utils import get_user_pref, save_user_pref +from netbox_librenms_plugin.utils import get_user_pref, save_user_pref, set_librenms_device_id from netbox_librenms_plugin.views.mixins import LibreNMSAPIMixin, LibreNMSPermissionMixin logger = logging.getLogger(__name__) @@ -153,6 +153,7 @@ def get_validated_device_with_selections(self, device_id: int, request) -> tuple include_vc_detection=enable_vc, use_sysname=use_sysname, strip_domain=strip_domain, + server_key=self.librenms_api.server_key, ) validation["import_as_vm"] = is_vm @@ -299,6 +300,7 @@ def post(self, request): api=self.librenms_api, use_sysname=use_sysname, strip_domain=strip_domain, + server_key=self.librenms_api.server_key, ) # Mark validation with VC detection flag for proper URL generation in table @@ -664,6 +666,7 @@ def post(self, request): # noqa: PLR0912 - branching keeps responses explicit import_as_vm=is_vm, api=None, # No VC detection needed for already-imported devices include_vc_detection=False, + server_key=self.librenms_api.server_key, ) validation["import_as_vm"] = is_vm @@ -918,7 +921,7 @@ def post(self, request, device_id): strip_domain=request.POST.get("strip-domain-toggle") == "on", ) ) - existing_device.custom_field_data["librenms_id"] = int(librenms_id) + set_librenms_device_id(existing_device, int(librenms_id), self.librenms_api.server_key) existing_device.name = hostname if librenms_device_type: existing_device.device_type = librenms_device_type @@ -938,7 +941,7 @@ def post(self, request, device_id): strip_domain=request.POST.get("strip-domain-toggle") == "on", ) ) - existing_device.custom_field_data["librenms_id"] = int(librenms_id) + set_librenms_device_id(existing_device, int(librenms_id), self.librenms_api.server_key) if incoming_serial and incoming_serial != "-": conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() if conflict_device: @@ -960,7 +963,7 @@ def post(self, request, device_id): elif action == "update_serial": # Update only the serial and link to LibreNMS incoming_serial = libre_device.get("serial") or "" - existing_device.custom_field_data["librenms_id"] = int(librenms_id) + set_librenms_device_id(existing_device, int(librenms_id), self.librenms_api.server_key) if incoming_serial and incoming_serial != "-": conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() if conflict_device: diff --git a/netbox_librenms_plugin/views/object_sync/devices.py b/netbox_librenms_plugin/views/object_sync/devices.py index 97584f8319..429a3119a8 100644 --- a/netbox_librenms_plugin/views/object_sync/devices.py +++ b/netbox_librenms_plugin/views/object_sync/devices.py @@ -79,13 +79,22 @@ def get_redirect_url(self, obj): def get_table(self, data, obj, interface_name_field, vlan_groups=None): """Return the appropriate interface table, selecting VC variant if needed.""" + server_key = self.librenms_api.server_key if hasattr(obj, "virtual_chassis") and obj.virtual_chassis: table = VCInterfaceTable( - data, device=obj, interface_name_field=interface_name_field, vlan_groups=vlan_groups + data, + device=obj, + interface_name_field=interface_name_field, + vlan_groups=vlan_groups, + server_key=server_key, ) else: table = LibreNMSInterfaceTable( - data, device=obj, interface_name_field=interface_name_field, vlan_groups=vlan_groups + data, + device=obj, + interface_name_field=interface_name_field, + vlan_groups=vlan_groups, + server_key=server_key, ) table.htmx_url = f"{self.request.path}?tab=interfaces" return table diff --git a/netbox_librenms_plugin/views/object_sync/vms.py b/netbox_librenms_plugin/views/object_sync/vms.py index 51143d909d..bd6e052488 100644 --- a/netbox_librenms_plugin/views/object_sync/vms.py +++ b/netbox_librenms_plugin/views/object_sync/vms.py @@ -45,7 +45,9 @@ class VMInterfaceTableView(BaseInterfaceTableView): def get_table(self, data, obj, interface_name_field, vlan_groups=None): """Return a VM interface table for the given data.""" - return LibreNMSVMInterfaceTable(data, device=obj, vlan_groups=vlan_groups) + return LibreNMSVMInterfaceTable( + data, device=obj, vlan_groups=vlan_groups, server_key=self.librenms_api.server_key + ) def get_interfaces(self, obj): """Return all interfaces for the virtual machine.""" diff --git a/netbox_librenms_plugin/views/sync/interfaces.py b/netbox_librenms_plugin/views/sync/interfaces.py index 1100e0da1b..e4bd1ad547 100644 --- a/netbox_librenms_plugin/views/sync/interfaces.py +++ b/netbox_librenms_plugin/views/sync/interfaces.py @@ -9,7 +9,7 @@ from virtualization.models import VirtualMachine, VMInterface from netbox_librenms_plugin.models import InterfaceTypeMapping -from netbox_librenms_plugin.utils import convert_speed_to_kbps, get_interface_name_field +from netbox_librenms_plugin.utils import convert_speed_to_kbps, get_interface_name_field, set_librenms_device_id from netbox_librenms_plugin.views.mixins import ( CacheMixin, LibreNMSPermissionMixin, @@ -235,7 +235,7 @@ def update_interface_attributes( setattr(interface, netbox_key, librenms_interface.get(librenms_key)) if "librenms_id" in interface.cf: - interface.custom_field_data["librenms_id"] = librenms_interface.get("port_id") + set_librenms_device_id(interface, librenms_interface.get("port_id"), self.librenms_api.server_key) if "enabled" not in exclude_columns: admin_status = librenms_interface.get("ifAdminStatus") From b9a44b1ee74f6492f4cc6c7d44ff374d520362a8 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sat, 28 Feb 2026 22:55:32 +0100 Subject: [PATCH 09/27] fix: address librenms_id-related PR review comments - utils.py set_librenms_device_id: defensively handle unexpected types (non-int, non-dict) by resetting to empty dict with a warning log - utils.py find_by_librenms_id: add OR query to also match legacy records where librenms_id is stored as a bare integer - cables_view.py: update all 6 librenms_id lookup sites to use Q()|Q() so legacy integer records are matched alongside new JSON format - actions.py: validate librenms_id (device_id) before int() cast and return 400 if missing/non-numeric; use pre-validated int throughout the link/update/update_serial action blocks - interfaces.py: guard against None port_id before set_librenms_device_id to avoid clobbering an existing server-key mapping --- netbox_librenms_plugin/utils.py | 22 ++++++++++++++++++- .../views/base/cables_view.py | 21 +++++++++++++----- .../views/imports/actions.py | 10 ++++++--- .../views/sync/interfaces.py | 4 +++- 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/netbox_librenms_plugin/utils.py b/netbox_librenms_plugin/utils.py index aae146b187..fee1d74d4e 100644 --- a/netbox_librenms_plugin/utils.py +++ b/netbox_librenms_plugin/utils.py @@ -9,6 +9,13 @@ from utilities.paginator import get_paginate_count as netbox_get_paginate_count +import logging + +from django.db.models import Q + +logger = logging.getLogger(__name__) + + def get_librenms_device_id(obj, server_key: str = "default"): """ Get the LibreNMS device/port ID for a specific server from the JSON custom field. @@ -49,6 +56,13 @@ def set_librenms_device_id(obj, device_id, server_key: str = "default"): cf_value = obj.custom_field_data.get("librenms_id") or {} if isinstance(cf_value, int): cf_value = {"default": cf_value} # migrate legacy value on first write + elif not isinstance(cf_value, dict): + logger.warning( + "librenms_id custom field has unexpected type %s on %r; resetting to empty dict.", + type(cf_value).__name__, + obj, + ) + cf_value = {} cf_value[server_key] = device_id obj.custom_field_data["librenms_id"] = cf_value @@ -58,6 +72,9 @@ def find_by_librenms_id(model, librenms_id, server_key: str = "default"): Return the first object of *model* whose ``librenms_id`` JSON field contains *librenms_id* under *server_key*. + Also matches legacy records that stored ``librenms_id`` as a bare integer + directly in ``custom_field_data``. + Args: model: A Django model class (Device, VirtualMachine, Interface, …). librenms_id: The LibreNMS device/port ID to look up. @@ -66,7 +83,10 @@ def find_by_librenms_id(model, librenms_id, server_key: str = "default"): Returns: Model instance or None """ - return model.objects.filter(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}).first() + return model.objects.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}) + | Q(custom_field_data__librenms_id=librenms_id) + ).first() def convert_speed_to_kbps(speed_bps: int) -> int: diff --git a/netbox_librenms_plugin/views/base/cables_view.py b/netbox_librenms_plugin/views/base/cables_view.py index 89c4a72a0e..4cbcdb7a36 100644 --- a/netbox_librenms_plugin/views/base/cables_view.py +++ b/netbox_librenms_plugin/views/base/cables_view.py @@ -4,6 +4,7 @@ from django.contrib import messages from django.core.cache import cache from django.core.exceptions import MultipleObjectsReturned +from django.db.models import Q from django.http import JsonResponse from django.shortcuts import get_object_or_404, render from django.urls import reverse @@ -82,7 +83,10 @@ def get_device_by_id_or_name(self, remote_device_id, hostname): # First try matching by LibreNMS ID if remote_device_id: try: - device = Device.objects.get(**{f"custom_field_data__librenms_id__{server_key}": remote_device_id}) + device = Device.objects.get( + Q(**{f"custom_field_data__librenms_id__{server_key}": remote_device_id}) + | Q(custom_field_data__librenms_id=remote_device_id) + ) return device, True, None except Device.DoesNotExist: pass @@ -125,7 +129,8 @@ def enrich_local_port(self, link, obj): # First try to find interface by librenms_id if local_port_id: interface = chassis_member.interfaces.filter( - **{f"custom_field_data__librenms_id__{server_key}": local_port_id} + Q(**{f"custom_field_data__librenms_id__{server_key}": local_port_id}) + | Q(custom_field_data__librenms_id=local_port_id) ).first() # Only if librenms_id match fails, try matching by name @@ -135,7 +140,8 @@ def enrich_local_port(self, link, obj): # First try to find interface by librenms_id if local_port_id: interface = obj.interfaces.filter( - **{f"custom_field_data__librenms_id__{server_key}": local_port_id} + Q(**{f"custom_field_data__librenms_id__{server_key}": local_port_id}) + | Q(custom_field_data__librenms_id=local_port_id) ).first() # Only if librenms_id match fails, try matching by name @@ -161,7 +167,8 @@ def enrich_remote_port(self, link, device): # First try to find interface by librenms_id if librenms_remote_port_id: netbox_remote_interface = chassis_member.interfaces.filter( - **{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id} + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id}) + | Q(custom_field_data__librenms_id=librenms_remote_port_id) ).first() # If not found by librenms_id, fall back to name matching on the correct chassis member @@ -172,7 +179,8 @@ def enrich_remote_port(self, link, device): # First try to find interface by librenms_id if librenms_remote_port_id: netbox_remote_interface = device.interfaces.filter( - **{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id} + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id}) + | Q(custom_field_data__librenms_id=librenms_remote_port_id) ).first() # If not found by librenms_id, fall back to name matching @@ -378,7 +386,8 @@ def post(self, request): if local_port_id := link_data.get("local_port_id"): _sk = self.librenms_api.server_key interface = selected_device.interfaces.filter( - **{f"custom_field_data__librenms_id__{_sk}": local_port_id} + Q(**{f"custom_field_data__librenms_id__{_sk}": local_port_id}) + | Q(custom_field_data__librenms_id=local_port_id) ).first() # If not found by librenms_id, try matching by name diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index b0e262a80a..8da9da8b0b 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -894,6 +894,10 @@ def post(self, request, device_id): librenms_device_type = validation.get("device_type", {}).get("device_type") librenms_id = libre_device.get("device_id") + try: + librenms_id = int(librenms_id) + except (TypeError, ValueError): + return HttpResponse("Invalid or missing LibreNMS device_id in payload", status=400) # Check for LibreNMS ID collision before any linking action if action in {"link", "update", "update_serial"}: @@ -921,7 +925,7 @@ def post(self, request, device_id): strip_domain=request.POST.get("strip-domain-toggle") == "on", ) ) - set_librenms_device_id(existing_device, int(librenms_id), self.librenms_api.server_key) + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) existing_device.name = hostname if librenms_device_type: existing_device.device_type = librenms_device_type @@ -941,7 +945,6 @@ def post(self, request, device_id): strip_domain=request.POST.get("strip-domain-toggle") == "on", ) ) - set_librenms_device_id(existing_device, int(librenms_id), self.librenms_api.server_key) if incoming_serial and incoming_serial != "-": conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() if conflict_device: @@ -954,6 +957,7 @@ def post(self, request, device_id): existing_device.name = hostname if librenms_device_type: existing_device.device_type = librenms_device_type + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) existing_device.save() logger.info( f"Updated device '{existing_device.name}': serial={incoming_serial}, " @@ -963,7 +967,6 @@ def post(self, request, device_id): elif action == "update_serial": # Update only the serial and link to LibreNMS incoming_serial = libre_device.get("serial") or "" - set_librenms_device_id(existing_device, int(librenms_id), self.librenms_api.server_key) if incoming_serial and incoming_serial != "-": conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() if conflict_device: @@ -975,6 +978,7 @@ def post(self, request, device_id): existing_device.serial = incoming_serial if librenms_device_type: existing_device.device_type = librenms_device_type + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) existing_device.save() logger.info( f"Updated serial on device '{existing_device.name}' to {incoming_serial}, " diff --git a/netbox_librenms_plugin/views/sync/interfaces.py b/netbox_librenms_plugin/views/sync/interfaces.py index e4bd1ad547..62f7e9eb3a 100644 --- a/netbox_librenms_plugin/views/sync/interfaces.py +++ b/netbox_librenms_plugin/views/sync/interfaces.py @@ -235,7 +235,9 @@ def update_interface_attributes( setattr(interface, netbox_key, librenms_interface.get(librenms_key)) if "librenms_id" in interface.cf: - set_librenms_device_id(interface, librenms_interface.get("port_id"), self.librenms_api.server_key) + port_id = librenms_interface.get("port_id") + if port_id is not None: + set_librenms_device_id(interface, port_id, self.librenms_api.server_key) if "enabled" not in exclude_columns: admin_status = librenms_interface.get("ifAdminStatus") From 16dfb658781ba7f1cdc77a40d1cef91d2b41d830 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sat, 28 Feb 2026 22:57:35 +0100 Subject: [PATCH 10/27] fix: update device_filter mocks to accept Q positional args find_by_librenms_id now calls .filter(Q(...)|Q(...)) passing a positional Q object. The three TestSerialNumberMatching mocks only accepted **kwargs, causing TypeError which was silently caught by the except block in device_operations.py, making existing_device None. Update all three device_filter side_effects to accept *args and check str(arg) for 'librenms_id' so Q-based lookups are detected correctly. --- .../tests/test_import_utils.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index a537f95d1c..0f0b7a5c39 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1344,9 +1344,12 @@ def test_librenms_id_match_shows_serial_confirmed(self): self.mock_vm.objects.filter.return_value.first.return_value = None - def device_filter(**kwargs): + def device_filter(*args, **kwargs): result = MagicMock() - if any(k.startswith("custom_field_data__librenms_id") for k in kwargs): + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + if q_has_librenms: result.first.return_value = existing else: result.first.return_value = None @@ -1384,9 +1387,12 @@ def test_librenms_id_match_detects_serial_drift(self): self.mock_vm.objects.filter.return_value.first.return_value = None - def device_filter(**kwargs): + def device_filter(*args, **kwargs): result = MagicMock() - if any(k.startswith("custom_field_data__librenms_id") for k in kwargs): + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + if q_has_librenms: result.first.return_value = existing elif "serial" in kwargs: result.first.return_value = None @@ -1426,9 +1432,12 @@ def test_librenms_id_match_still_validates_site(self): self.mock_vm.objects.filter.return_value.first.return_value = None - def device_filter(**kwargs): + def device_filter(*args, **kwargs): result = MagicMock() - if any(k.startswith("custom_field_data__librenms_id") for k in kwargs): + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + if q_has_librenms: result.first.return_value = existing else: result.first.return_value = None From e0e9c406b15c3fb9a37ffa8d8e50bb1f96a40b76 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Sun, 1 Mar 2026 23:29:08 +0100 Subject: [PATCH 11/27] =?UTF-8?q?feat:=20legacy=20librenms=5Fid=20int?= =?UTF-8?q?=E2=86=92JSON=20migration=20action=20on=20import=20page?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add migrate_legacy_librenms_id(obj, server_key) helper in utils.py Converts a bare-integer librenms_id to {server_key: int_value}. Returns True if migration happened, False if already JSON or absent. Does not call save() — caller is responsible. - Detect legacy int format in validate_device_for_import() When find_by_librenms_id matches a Device or VM whose librenms_id CF is still a bare integer, set result['librenms_id_needs_migration']=True so the import page can surface a migration action. - Add 'migrate_librenms_id' action in DeviceConflictActionView Verifies CF is still an int, requires serial_confirmed or force checkbox, calls migrate_legacy_librenms_id() + save(). - Fix collision check to use find_by_librenms_id instead of raw queryset The old Device.objects.filter(custom_field_data__librenms_id=int(...)) only matched the legacy integer format; the new helper matches both the JSON dict format and the legacy format. - Show 'Legacy ID format' badge + 'Migrate ID format' button in template Appears in the existing_match_type=='librenms_id' section when librenms_id_needs_migration is set. Button is disabled (requires force checkbox) when serial not confirmed. - Add TestLegacyLibreNMSIdMigration test class (6 tests) - Update TestDeviceConflictActionView mocks for new filter().first() chain --- .../import_utils/device_operations.py | 9 ++ .../htmx/device_validation_details.html | 26 +++ .../tests/test_import_utils.py | 151 ++++++++++++++++++ netbox_librenms_plugin/utils.py | 33 ++++ .../views/imports/actions.py | 34 +++- 5 files changed, 247 insertions(+), 6 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index c21d0bfb7a..11f058a895 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -160,6 +160,7 @@ def validate_device_for_import( "serial_action": None, # None, "link", "conflict", "update_serial", "hostname_differs" "serial_confirmed": False, # True when librenms_id match and serial matches "serial_duplicate": False, # True when incoming serial is already on a different device + "librenms_id_needs_migration": False, # True when librenms_id is still a legacy bare int "name_matches": False, # True when existing device name matches LibreNMS sysName "name_sync_available": False, # True when existing device name differs from sysName "suggested_name": None, # sysName to suggest when name_sync_available is True @@ -232,6 +233,10 @@ def validate_device_for_import( result["import_as_vm"] = True # Force VM mode since VM exists result["can_import"] = False + # Detect legacy bare-integer format so UI can offer a migration action + if isinstance(existing_vm.custom_field_data.get("librenms_id"), int): + result["librenms_id_needs_migration"] = True + # Check if name matches resolved name (accounts for use_sysname/strip_domain) # Note: name_sync_available/suggested_name are intentionally not set for VMs # because UpdateDeviceNameView only supports Device objects; VM name-sync @@ -255,6 +260,10 @@ def validate_device_for_import( result["existing_match_type"] = "librenms_id" result["can_import"] = False + # Detect legacy bare-integer format so UI can offer a migration action + if isinstance(existing_device.custom_field_data.get("librenms_id"), int): + result["librenms_id_needs_migration"] = True + # Check if name matches resolved name (accounts for use_sysname/strip_domain) if hostname and existing_device.name == hostname: result["name_matches"] = True diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html index 2bb7fb7f8c..4eee668e85 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html @@ -376,7 +376,33 @@
Device Information
{% if validation.device_type_mismatch %} Type mismatch {% endif %} + {% if validation.librenms_id_needs_migration %} + Legacy ID format + {% endif %}
+ {% if validation.librenms_id_needs_migration %} +
+ + {% csrf_token %} + + + {% if not validation.serial_confirmed %} +
+ + +
+ {% endif %} + + +
+ {% endif %} {% elif validation.existing_match_type == 'hostname' %}
diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 0f0b7a5c39..1b8df20727 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1614,6 +1614,151 @@ def device_filter(**kwargs): assert result["device_type_mismatch"] is False +class TestLegacyLibreNMSIdMigration: + """Test detection of legacy bare-integer librenms_id format during device validation.""" + + PATCHES = [ + "netbox_librenms_plugin.import_utils.device_operations.Site", + "netbox_librenms_plugin.import_utils.device_operations.Rack", + "netbox_librenms_plugin.import_utils.device_operations.Cluster", + "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", + "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", + "netbox_librenms_plugin.import_utils.device_operations.Device", + "virtualization.models.VirtualMachine", + ] + + def setup_method(self): + self._patchers = [patch(p) for p in self.PATCHES] + mocks = [p.start() for p in self._patchers] + ( + self.mock_site_model, + self.mock_rack, + self.mock_cluster, + self.mock_role, + self.mock_match_type, + self.mock_find_platform, + self.mock_find_site, + self.mock_device, + self.mock_vm, + ) = mocks + + self.mock_find_site.return_value = { + "found": True, + "site": MagicMock(), + "match_type": "exact", + "confidence": 1.0, + } + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_rack.objects.filter.return_value = [] + self.mock_site_model.objects.all.return_value = [] + self.mock_vm.objects.filter.return_value.first.return_value = None + + def teardown_method(self): + for p in self._patchers: + p.stop() + + def _make_existing(self, librenms_id_value, serial="SN001"): + existing = MagicMock() + existing.name = "switch-01" + existing.serial = serial + existing.custom_field_data = {"librenms_id": librenms_id_value} + return existing + + def _setup_device_filter(self, existing): + def device_filter(*args, **kwargs): + result = MagicMock() + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + result.first.return_value = existing if q_has_librenms else None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + def test_legacy_int_sets_needs_migration_flag(self): + """Device with bare-integer librenms_id sets librenms_id_needs_migration=True.""" + existing = self._make_existing(librenms_id_value=42, serial="SN001") + self._setup_device_filter(existing) + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + result = validate_device_for_import( + {"device_id": 42, "hostname": "switch-01", "serial": "SN001"}, + include_vc_detection=False, + ) + + assert result["existing_match_type"] == "librenms_id" + assert result["librenms_id_needs_migration"] is True + assert result["serial_confirmed"] is True + + def test_legacy_int_no_serial_still_sets_flag(self): + """Legacy int format sets the migration flag even when serial is absent.""" + existing = self._make_existing(librenms_id_value=42, serial="") + self._setup_device_filter(existing) + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + result = validate_device_for_import( + {"device_id": 42, "hostname": "switch-01"}, + include_vc_detection=False, + ) + + assert result["librenms_id_needs_migration"] is True + assert result["serial_confirmed"] is False + + def test_json_format_does_not_set_flag(self): + """Device with JSON librenms_id does NOT set librenms_id_needs_migration.""" + existing = self._make_existing(librenms_id_value={"default": 42}, serial="SN001") + self._setup_device_filter(existing) + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + result = validate_device_for_import( + {"device_id": 42, "hostname": "switch-01", "serial": "SN001"}, + include_vc_detection=False, + ) + + assert result["existing_match_type"] == "librenms_id" + assert result["librenms_id_needs_migration"] is False + + def test_migrate_legacy_librenms_id_helper(self): + """migrate_legacy_librenms_id converts int to {server_key: int}.""" + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + obj = MagicMock() + obj.custom_field_data = {"librenms_id": 42} + result = migrate_legacy_librenms_id(obj, "primary") + + assert result is True + assert obj.custom_field_data["librenms_id"] == {"primary": 42} + + def test_migrate_legacy_librenms_id_noop_for_json(self): + """migrate_legacy_librenms_id is a no-op when value is already a dict.""" + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + obj = MagicMock() + obj.custom_field_data = {"librenms_id": {"primary": 42}} + result = migrate_legacy_librenms_id(obj, "primary") + + assert result is False + assert obj.custom_field_data["librenms_id"] == {"primary": 42} + + def test_migrate_legacy_librenms_id_noop_for_none(self): + """migrate_legacy_librenms_id is a no-op when librenms_id is absent.""" + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + obj = MagicMock() + obj.custom_field_data = {} + result = migrate_legacy_librenms_id(obj, "primary") + + assert result is False + + class TestDeviceConflictActionView: """Test DeviceConflictActionView conflict resolution actions.""" @@ -1668,6 +1813,7 @@ def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache) patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1708,6 +1854,7 @@ def test_update_action_sets_hostname_serial_and_librenms_id(self, mock_cache_key patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1744,6 +1891,7 @@ def test_update_serial_action_updates_serial_only(self, mock_cache_key, mock_cac patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1915,6 +2063,7 @@ def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cach patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1960,6 +2109,7 @@ def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cach patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -2036,6 +2186,7 @@ def test_sync_serial_action(self, mock_cache_key, mock_cache): patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() diff --git a/netbox_librenms_plugin/utils.py b/netbox_librenms_plugin/utils.py index fee1d74d4e..62dedcd01b 100644 --- a/netbox_librenms_plugin/utils.py +++ b/netbox_librenms_plugin/utils.py @@ -89,6 +89,39 @@ def find_by_librenms_id(model, librenms_id, server_key: str = "default"): ).first() +def migrate_legacy_librenms_id(obj, server_key: str = "default") -> bool: + """ + Migrate a legacy bare-integer ``librenms_id`` custom field to the JSON dict format, + scoped to *server_key*. + + Only performs the migration when the current value is a bare integer, i.e. a record + created before the multi-server JSON refactor. The integer is assumed to belong to + the server identified by *server_key* (the caller must verify this, e.g. by confirming + that the LibreNMS device ID and serial number both match). + + Does **not** call ``obj.save()`` — the caller is responsible for persisting the change. + + Args: + obj: NetBox object with a ``librenms_id`` custom field. + server_key: LibreNMS server key the legacy integer should be scoped to. + + Returns: + True if the value was migrated, False if it was already in the correct format. + """ + cf_value = obj.custom_field_data.get("librenms_id") + if not isinstance(cf_value, int): + return False + obj.custom_field_data["librenms_id"] = {server_key: cf_value} + logger.info( + "Migrated legacy librenms_id %d → {%r: %d} on %r", + cf_value, + server_key, + cf_value, + obj, + ) + return True + + def convert_speed_to_kbps(speed_bps: int) -> int: """ Convert speed from bits per second to kilobits per second. diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 8da9da8b0b..d5504dae52 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -901,12 +901,10 @@ def post(self, request, device_id): # Check for LibreNMS ID collision before any linking action if action in {"link", "update", "update_serial"}: - id_conflict = ( - Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)) - .exclude(pk=existing_device.pk) - .first() - ) - if id_conflict: + from netbox_librenms_plugin.utils import find_by_librenms_id + + id_conflict = find_by_librenms_id(Device, librenms_id, self.librenms_api.server_key) + if id_conflict and id_conflict.pk != existing_device.pk: return HttpResponse( f"LibreNMS ID conflict: ID {librenms_id} is already assigned to device " f"'{id_conflict.name}' (ID: {id_conflict.pk})", @@ -1061,6 +1059,30 @@ def post(self, request, device_id): else: return HttpResponse(f"No matching device type for '{hardware}'", status=400) + elif action == "migrate_librenms_id": + # Migrate legacy bare-integer librenms_id to the JSON dict format. + # Only safe when the integer matches the LibreNMS device ID for this server, + # confirmed by serial match (or explicit force). + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + cf_value = existing_device.custom_field_data.get("librenms_id") + if not isinstance(cf_value, int): + return HttpResponse( + "Device librenms_id is already in JSON format; no migration needed.", + status=400, + ) + if not validation.get("serial_confirmed") and not force: + return HttpResponse( + "Serial number not confirmed. Check the force checkbox to migrate without serial verification.", + status=400, + ) + migrate_legacy_librenms_id(existing_device, self.librenms_api.server_key) + existing_device.save() + logger.info( + f"Migrated legacy librenms_id on '{existing_device.name}' " + f"to {{{self.librenms_api.server_key!r}: {cf_value}}}" + ) + else: return HttpResponse(f"Unknown action: {action}", status=400) From c439c45a66a63ed23705100ba1c9294337ee8172 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 00:00:16 +0100 Subject: [PATCH 12/27] =?UTF-8?q?fix:=20PR=20review=20fixes=20=E2=80=94=20?= =?UTF-8?q?schema,=20server=5Fkey,=20bulk=5Fimport,=20legacy=20badge,=20ac?= =?UTF-8?q?cessibility?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - device_operations.py: preserve device_type schema dict instead of replacing with raw dt_match; 'found' key is now always set regardless of match result - device_operations.py: use api.server_key instead of server_key or 'default' to consistently use the resolved server key after LibreNMSAPI instantiation - device_operations.py: move Q import to module level; remove inline 'from dcim.models import Rack' so test mock patching works correctly - bulk_import.py: fix 7 early-return sites to return ([], False) when return_cache_status=True, keeping the return shape consistent for callers - device_status.py: add 'Legacy ID' warning badge when librenms_id_needs_migration is set on an existing device matched by librenms_id - device_status.py: add aria-label to icon-only buttons for accessibility - device_validation_details.html: add rel='noopener noreferrer' to all target='_blank' anchors (8 links) - test_import_utils.py: add DeviceType to SERIAL_PATCHES so suggestions path is mocked; fix _create_view to use real instantiation; fix _setup_no_existing to use named variables instead of fragile negative indexing; remove dead mock_rack.objects.filter.return_value = [] lines (inline import bypass meant these were never effective); fix device_type assertions to use 'found' key --- .../import_utils/bulk_import.py | 14 +++--- .../import_utils/device_operations.py | 18 +++---- .../tables/device_status.py | 6 +++ .../htmx/device_validation_details.html | 16 +++--- .../tests/test_import_utils.py | 49 +++++++------------ 5 files changed, 45 insertions(+), 58 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 0409b35da3..66d48de6a7 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -383,7 +383,7 @@ def process_device_filters( except (BrokenPipeError, ConnectionError, IOError) as e: if request: logger.info(f"Client disconnected during VC prefetch: {e}") - return [] + return ([], False) if return_cache_status else [] raise # Validate each device @@ -403,13 +403,13 @@ def process_device_filters( if rq_job.is_failed or rq_job.is_stopped: job.logger.warning("Job was already stopped before validation started") - return [] + return ([], False) if return_cache_status else [] except Exception: # Fall back to DB check if RQ check fails job.job.refresh_from_db() if job.job.status == JobStatusChoices.STATUS_FAILED: job.logger.warning("Job was stopped before validation started") - return [] + return ([], False) if return_cache_status else [] else: logger.info(f"Validating {total} devices") @@ -432,13 +432,13 @@ def process_device_filters( job.logger.info( f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." ) - return [] + return ([], False) if return_cache_status else [] except Exception: # If we can't check RQ status, fall back to DB status check job.job.refresh_from_db() if job.job.status == JobStatusChoices.STATUS_FAILED: job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") - return [] + return ([], False) if return_cache_status else [] elif request: # Check for client disconnect try: @@ -446,7 +446,7 @@ def process_device_filters( pass except (BrokenPipeError, ConnectionError, IOError): logger.info(f"Client disconnected during validation at device {idx}") - return [] + return ([], False) if return_cache_status else [] # Drop any cached validation/meta keys before recomputing device.pop("_validation", None) @@ -493,7 +493,7 @@ def process_device_filters( except (BrokenPipeError, ConnectionError, IOError) as e: if request: logger.info(f"Client disconnected during device validation: {e}") - return [] + return ([], False) if return_cache_status else [] raise # Set VC detection metadata diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index 11f058a895..dab87b37d0 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -5,6 +5,7 @@ from dcim.models import Device, DeviceRole, DeviceType, Rack, Site from django.core.cache import cache from django.db import transaction +from django.db.models import Q from django.utils import timezone from virtualization.models import Cluster # noqa: F401 — used by test mock.patch targets @@ -435,7 +436,10 @@ def validate_device_for_import( # 3. Validate DeviceType (required) hardware = libre_device.get("hardware", "") dt_match = match_librenms_hardware_to_device_type(hardware) - result["device_type"] = dt_match + # Update result keys individually to preserve the existing schema (especially "found") + result["device_type"]["found"] = dt_match["matched"] + result["device_type"]["device_type"] = dt_match.get("device_type") + result["device_type"]["match_type"] = dt_match.get("match_type") if not dt_match["matched"]: result["issues"].append(f"No matching device type found for hardware: '{hardware}'") @@ -449,11 +453,6 @@ def validate_device_for_import( } for dt in all_device_types ] - else: - # Rename 'matched' to 'found' for consistency - result["device_type"]["found"] = dt_match["matched"] - result["device_type"]["device_type"] = dt_match["device_type"] - result["device_type"]["match_type"] = dt_match["match_type"] # 4. DeviceRole (required) - Must be manually selected by user logger.debug(f"[{hostname}] Issues BEFORE adding role issue: {result['issues']}") @@ -478,9 +477,6 @@ def validate_device_for_import( available_racks = cache.get(cache_key) if available_racks is None: - from dcim.models import Rack - from django.db.models import Q - # Query racks for this site - include both: # 1. Racks assigned to locations within the site # 2. Racks directly assigned to the site (without location) @@ -652,7 +648,7 @@ def import_single_device( libre_device, use_sysname=use_sysname_opt, strip_domain=strip_domain_opt, - server_key=server_key or "default", + server_key=api.server_key, ) # Check if device already exists @@ -738,7 +734,7 @@ def import_single_device( "role": device_role, "status": "active" if libre_device.get("status") == 1 else "offline", "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - "custom_field_data": {"librenms_id": {(server_key or "default"): int(device_id)}}, + "custom_field_data": {"librenms_id": {api.server_key: int(device_id)}}, } # Add optional fields diff --git a/netbox_librenms_plugin/tables/device_status.py b/netbox_librenms_plugin/tables/device_status.py index 3a548bcd96..58eeb7c2ad 100644 --- a/netbox_librenms_plugin/tables/device_status.py +++ b/netbox_librenms_plugin/tables/device_status.py @@ -479,15 +479,21 @@ def render_actions(self, value, record): btn_class = "btn-outline-warning" btn_icon = "mdi-information-outline" btn_label = " Details" + elif match_type == "librenms_id" and validation.get("librenms_id_needs_migration"): + btn_class = "btn-outline-warning" + btn_icon = "mdi-database-alert" + btn_label = " Legacy ID" else: btn_class = "btn-outline-success" btn_icon = "mdi-check-circle" btn_label = "" btn_title = "Resolve conflict" if (has_actions or has_mismatch) else "View details" + aria_attr = f'aria-label="{btn_title}" ' if btn_label == "" else "" buttons.append( f'
@@ -479,7 +479,7 @@
Device Information
{% endif %} — Exists as - {{ validation.existing_device.name }}, + {{ validation.existing_device.name }}, not linked to LibreNMS. @@ -517,7 +517,7 @@
Device Information
IP match — Device with IP {{ libre_device.ip }} exists as - {{ validation.existing_device.name }}. + {{ validation.existing_device.name }}. Consider adding LibreNMS ID manually. @@ -526,7 +526,7 @@
Device Information
Exists — Device already exists as - {{ validation.existing_device.name }}. + {{ validation.existing_device.name }}.
{% endif %} @@ -569,17 +569,17 @@
Device Information
{% if validation.existing_device %} {% if validation.import_as_vm or validation.existing_device.cluster %} + class="btn btn-primary btn-sm" target="_blank" rel="noopener noreferrer"> View VM in NetBox {% else %} + class="btn btn-primary btn-sm" target="_blank" rel="noopener noreferrer"> View in NetBox {% if validation.existing_match_type == 'librenms_id' %} + class="btn btn-outline-primary btn-sm" target="_blank" rel="noopener noreferrer"> Full Sync Page {% endif %} diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 1b8df20727..b2c992be74 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -341,7 +341,6 @@ def test_validate_device_site_match_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [mock_site] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -399,7 +398,6 @@ def test_validate_device_site_not_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -461,7 +459,6 @@ def test_validate_device_platform_match_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -519,7 +516,6 @@ def test_validate_device_platform_not_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -577,7 +573,6 @@ def test_validate_device_type_match_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -638,7 +633,6 @@ def test_validate_device_type_not_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -651,7 +645,7 @@ def test_validate_device_type_not_found( result = validate_device_for_import(device_data, include_vc_detection=False) - assert result["device_type"]["matched"] is False + assert result["device_type"]["found"] is False assert any("device type" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") @@ -698,7 +692,6 @@ def test_validate_device_role_required( } mock_role.objects.all.return_value = [MagicMock(id=1, name="Access Switch")] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [mock_site] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -757,7 +750,6 @@ def test_validate_device_handles_empty_location( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -816,7 +808,6 @@ def test_validate_device_handles_empty_os( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -877,7 +868,6 @@ def test_validate_device_handles_empty_hardware( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -891,7 +881,7 @@ def test_validate_device_handles_empty_hardware( result = validate_device_for_import(device_data, include_vc_detection=False) assert result is not None - assert result["device_type"]["matched"] is False + assert result["device_type"]["found"] is False @patch("virtualization.models.VirtualMachine") @patch("netbox_librenms_plugin.import_utils.device_operations.Device") @@ -974,7 +964,6 @@ def test_validate_device_returns_complete_state( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1045,7 +1034,6 @@ def test_validate_device_import_as_vm( mock_clusters = [MagicMock(id=1, name="VMware Cluster")] mock_cluster.objects.all.return_value = mock_clusters mock_cache.get.return_value = None # Force cache miss to trigger Cluster.objects.all() - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1109,6 +1097,7 @@ class TestSerialNumberMatching: "netbox_librenms_plugin.import_utils.device_operations.Rack", "netbox_librenms_plugin.import_utils.device_operations.Cluster", "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", + "netbox_librenms_plugin.import_utils.device_operations.DeviceType", "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", @@ -1125,12 +1114,14 @@ def _start_patches(self): self.mock_rack, self.mock_cluster, self.mock_role, + self.mock_device_type, self.mock_match_type, self.mock_find_platform, self.mock_find_site, self.mock_device, self.mock_vm, ) = mocks + self.mock_device_type.objects.all.return_value = [] def _stop_patches(self): """Stop all patches.""" @@ -1265,7 +1256,6 @@ def _setup_no_match_mocks(self): self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] def test_serial_dash_ignored(self): @@ -1366,7 +1356,6 @@ def device_filter(*args, **kwargs): self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1412,7 +1401,6 @@ def device_filter(*args, **kwargs): self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1451,7 +1439,6 @@ def device_filter(*args, **kwargs): self.mock_match_type.return_value = {"matched": True, "device_type": mock_dt, "match_type": "exact"} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1492,7 +1479,6 @@ def device_filter(**kwargs): self.mock_match_type.return_value = {"matched": True, "device_type": MagicMock(), "match_type": "exact"} self.mock_role.objects.all.return_value = [mock_existing_role] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: @@ -1548,7 +1534,6 @@ def device_filter(**kwargs): } self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: @@ -1594,7 +1579,6 @@ def device_filter(**kwargs): self.mock_match_type.return_value = {"matched": True, "device_type": same_device_type, "match_type": "exact"} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: @@ -1654,7 +1638,6 @@ def setup_method(self): self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] self.mock_vm.objects.filter.return_value.first.return_value = None @@ -1766,7 +1749,7 @@ def _create_view(self): """Create a DeviceConflictActionView instance with mocked dependencies.""" from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView - view = object.__new__(DeviceConflictActionView) + view = DeviceConflictActionView() view._librenms_api = MagicMock() view._librenms_api.server_key = "default" view.request = MagicMock() @@ -2363,14 +2346,17 @@ class TestDeviceNamingPreferences: def _setup_no_existing(self, mocks): """Configure mocks so no existing device is found.""" - mock_vm = mocks[-1] # VirtualMachine - mock_device = mocks[-2] # Device - mock_find_site = mocks[-3] - mock_find_platform = mocks[-4] - mock_match_type = mocks[-5] - mock_role = mocks[-6] - mock_rack = mocks[-8] - mock_site_model = mocks[-9] + ( + mock_site_model, + mock_rack, + mock_cluster, + mock_role, + mock_match_type, + mock_find_platform, + mock_find_site, + mock_device, + mock_vm, + ) = mocks mock_vm.objects.filter.return_value.first.return_value = None mock_device.objects.filter.return_value.first.return_value = None @@ -2391,7 +2377,6 @@ def _setup_no_existing(self, mocks): "match_type": None, } mock_role.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] @patch("virtualization.models.VirtualMachine") From 9068c0cbaf978ec6e5e85e5a4a32e74fc8b88c89 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 00:26:28 +0100 Subject: [PATCH 13/27] PR review: code quality, accessibility and correctness fixes - import_utils/cache.py: guard datetime.fromisoformat with try/except fallback; replace hash() with deterministic sha256 for cross-process stable cache keys - import_utils/filters.py: replace hash() with deterministic sha256 for librenms_devices_import cache key; add hashlib/json imports - import_utils/permissions.py: remove unused logger and logging import - import_utils/virtual_chassis.py: fix blank serial skip logic - only skip member when serial is non-empty AND matches master serial; same fix in expected_members count - device_validation_details.html: add aria-label to 6 icon-only sync buttons for screen-reader accessibility - test_import_utils.py: replace fragile mocks[-2] indexing with named unpacking - views/imports/actions.py: import escape(); HTML-escape untrusted values (id_conflict.name, librenms_os, hardware, action) in HttpResponse f-strings - views/sync/interfaces.py: guard handle_mac_address with is_device_interface to avoid setting MAC on VMInterface objects - import_utils/device_operations.py: reassign import_as_vm from result dict after existing-object detection so downstream branches use effective VM mode --- netbox_librenms_plugin/import_utils/cache.py | 14 +++++++++++--- .../import_utils/device_operations.py | 4 ++++ netbox_librenms_plugin/import_utils/filters.py | 11 +++++++++-- netbox_librenms_plugin/import_utils/permissions.py | 4 ---- .../import_utils/virtual_chassis.py | 8 +++++--- .../htmx/device_validation_details.html | 12 ++++++------ netbox_librenms_plugin/tests/test_import_utils.py | 13 ++++++++++++- netbox_librenms_plugin/views/imports/actions.py | 9 +++++---- netbox_librenms_plugin/views/sync/interfaces.py | 2 +- 9 files changed, 53 insertions(+), 24 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/cache.py b/netbox_librenms_plugin/import_utils/cache.py index 716e9dc6a1..7430d129f4 100644 --- a/netbox_librenms_plugin/import_utils/cache.py +++ b/netbox_librenms_plugin/import_utils/cache.py @@ -1,5 +1,7 @@ """Cache key generation and management for device import operations.""" +import hashlib +import json import logging from django.core.cache import cache @@ -71,9 +73,15 @@ def get_active_cached_searches(server_key: str) -> list[dict]: metadata = cache.get(cache_key) if metadata: # Cache still exists, calculate time remaining - cached_at = datetime.fromisoformat(metadata.get("cached_at")) cache_timeout = metadata.get("cache_timeout", 300) now = datetime.now(timezone.utc) + try: + cached_at_raw = metadata.get("cached_at") + cached_at = ( + datetime.fromisoformat(cached_at_raw) if cached_at_raw else datetime.fromtimestamp(0, timezone.utc) + ) + except (ValueError, TypeError): + cached_at = datetime.fromtimestamp(0, timezone.utc) age_seconds = (now - cached_at).total_seconds() remaining_seconds = max(0, cache_timeout - age_seconds) @@ -130,8 +138,8 @@ def get_validated_device_cache_key(server_key: str, filters: dict, device_id: in >>> key 'validated_device_default_-1234567890_123_vc' """ - # Sort filters for consistent hashing - filter_hash = hash(str(sorted(filters.items()))) + # Sort filters for a deterministic, cross-process stable hash + filter_hash = hashlib.sha256(json.dumps(sorted(filters.items()), sort_keys=True).encode()).hexdigest()[:16] vc_part = "vc" if vc_enabled else "novc" return f"validated_device_{server_key}_{filter_hash}_{device_id}_{vc_part}" diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index dab87b37d0..8113c31fad 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -400,6 +400,10 @@ def validate_device_for_import( ) result["can_import"] = False + # Refresh local variable to reflect any VM-mode adjustments made during detection + # (e.g. existing VM found by hostname sets result["import_as_vm"] = True) + import_as_vm = result["import_as_vm"] + # Validate based on import type (Device or VM) if import_as_vm: # 2. For VMs: Validate Cluster (required) - Must be manually selected diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py index 27f4449266..9f55576a26 100644 --- a/netbox_librenms_plugin/import_utils/filters.py +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -1,5 +1,7 @@ """Device filtering and retrieval from LibreNMS.""" +import hashlib +import json import logging from typing import List @@ -170,8 +172,13 @@ def get_librenms_devices_for_import( # We'll filter client-side if needed # Use caching to avoid repeated API calls - # Include both API and client filters in cache key - cache_key = f"librenms_devices_import_{server_key}_{hash(str(api_filters))}_{hash(str(client_filters))}" + # Include both API and client filters in cache key (deterministic, cross-process stable) + def _hash(d): + return hashlib.sha256( + json.dumps(sorted(d.items()) if isinstance(d, dict) else d, sort_keys=True).encode() + ).hexdigest()[:16] + + cache_key = f"librenms_devices_import_{server_key}_{_hash(api_filters)}_{_hash(client_filters)}" from_cache = False if force_refresh: diff --git a/netbox_librenms_plugin/import_utils/permissions.py b/netbox_librenms_plugin/import_utils/permissions.py index 9e9b4521e0..742e50893c 100644 --- a/netbox_librenms_plugin/import_utils/permissions.py +++ b/netbox_librenms_plugin/import_utils/permissions.py @@ -1,11 +1,7 @@ """Permission check helpers for device import operations.""" -import logging - from django.core.exceptions import PermissionDenied -logger = logging.getLogger(__name__) - def check_user_permissions(user, permissions): """ diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py index 79db61e3fe..4dc44e4182 100644 --- a/netbox_librenms_plugin/import_utils/virtual_chassis.py +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -373,8 +373,8 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis members_created = 0 for member in members_info: - # Skip if this is the master's serial - if member.get("serial") == master_device.serial: + # Skip if this is the master's serial (only when both serials are non-empty) + if member.get("serial") and member.get("serial") == master_device.serial: continue serial = member.get("serial") @@ -414,7 +414,9 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis position += 1 # Validate member count - expected_members = len([m for m in members_info if m.get("serial") != master_device.serial]) + expected_members = len( + [m for m in members_info if not (m.get("serial") and m.get("serial") == master_device.serial)] + ) if members_created < expected_members: logger.warning( f"Created {members_created} members but expected {expected_members}. " diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html index a758aa79b1..2f1ac5e70b 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html @@ -108,7 +108,7 @@
Device Information
{% csrf_token %} - @@ -156,7 +156,7 @@
Device Information
- @@ -169,7 +169,7 @@
Device Information
{% csrf_token %} - @@ -209,7 +209,7 @@
Device Information
{% csrf_token %} - @@ -263,7 +263,7 @@
Device Information
{% csrf_token %} - @@ -287,7 +287,7 @@
Device Information
{% csrf_token %} - diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index b2c992be74..27c157b1dd 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -2466,7 +2466,18 @@ def test_duplicate_detection_uses_resolved_name(self, *mocks): """Duplicate detection should match against the resolved name, not raw hostname.""" self._setup_no_existing(mocks) - mock_device = mocks[-2] # Device + # Unpack using same order as _setup_no_existing / @patch decorators (bottom-up) + ( + _mock_site, + _mock_rack, + _mock_cluster, + _mock_role, + _mock_hw, + _mock_platform, + _mock_find_site, + mock_device, + _mock_vm, + ) = mocks existing = MagicMock() existing.name = "core-switch" existing.serial = "" diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index d5504dae52..21682e6403 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -8,6 +8,7 @@ from django.core.exceptions import PermissionDenied from django.http import HttpResponse, JsonResponse from django.shortcuts import redirect, render +from django.utils.html import escape from django.views import View from netbox_librenms_plugin.import_utils import ( @@ -907,7 +908,7 @@ def post(self, request, device_id): if id_conflict and id_conflict.pk != existing_device.pk: return HttpResponse( f"LibreNMS ID conflict: ID {librenms_id} is already assigned to device " - f"'{id_conflict.name}' (ID: {id_conflict.pk})", + f"'{escape(id_conflict.name)}' (ID: {id_conflict.pk})", status=409, ) @@ -1042,7 +1043,7 @@ def post(self, request, device_id): existing_device.save() logger.info(f"Synced platform on '{existing_device.name}' to {match_result['platform']}") else: - return HttpResponse(f"Platform '{librenms_os}' not found in NetBox", status=400) + return HttpResponse(f"Platform '{escape(librenms_os)}' not found in NetBox", status=400) else: return HttpResponse("No OS info from LibreNMS", status=400) @@ -1057,7 +1058,7 @@ def post(self, request, device_id): existing_device.save() logger.info(f"Synced device type on '{existing_device.name}' to {hw_match['device_type']}") else: - return HttpResponse(f"No matching device type for '{hardware}'", status=400) + return HttpResponse(f"No matching device type for '{escape(hardware)}'", status=400) elif action == "migrate_librenms_id": # Migrate legacy bare-integer librenms_id to the JSON dict format. @@ -1084,7 +1085,7 @@ def post(self, request, device_id): ) else: - return HttpResponse(f"Unknown action: {action}", status=400) + return HttpResponse(f"Unknown action: {escape(action)}", status=400) # Clear cached validation so re-validation picks up the changes cache_key = get_import_device_cache_key(device_id, self.librenms_api.server_key) diff --git a/netbox_librenms_plugin/views/sync/interfaces.py b/netbox_librenms_plugin/views/sync/interfaces.py index 62f7e9eb3a..f5d92ab0a2 100644 --- a/netbox_librenms_plugin/views/sync/interfaces.py +++ b/netbox_librenms_plugin/views/sync/interfaces.py @@ -247,7 +247,7 @@ def update_interface_attributes( else (admin_status.lower() == "up" if isinstance(admin_status, str) else bool(admin_status)) ) - if "mac_address" not in exclude_columns: + if "mac_address" not in exclude_columns and is_device_interface: ifPhysAddress = librenms_interface.get("ifPhysAddress") self.handle_mac_address(interface, ifPhysAddress) From 6c3425f1e717f57613df6d8401cf95f498d6f5da Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 09:26:03 +0100 Subject: [PATCH 14/27] Fix _refresh_existing_device readiness logic for VMs When an existing device is deleted between cache and refresh, recompute can_import and is_ready to match validate_device_for_import semantics: - can_import = not bool(issues) (not unconditionally True) - VMs: is_ready = can_import AND cluster.found (not site+cluster+device_role) - Devices: is_ready = can_import AND site+device_type+device_role --- .../import_utils/bulk_import.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 66d48de6a7..848cc03d05 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -287,22 +287,23 @@ def _refresh_existing_device(validation: dict) -> None: validation["device_role"]["found"] = True validation["device_role"]["role"] = refreshed.role else: - # Device was deleted since caching — recompute readiness + # Device was deleted since caching — recompute readiness to match + # validate_device_for_import logic. validation["existing_device"] = None validation["existing_match_type"] = None + can_import = not bool(validation.get("issues")) if validation.get("import_as_vm"): - required_found = ( - validation.get("site", {}).get("found") - and validation.get("cluster", {}).get("found") - and validation.get("device_role", {}).get("found") - ) + # VMs only require a cluster (site/role not mandatory) + is_ready = can_import and bool(validation.get("cluster", {}).get("found")) else: - required_found = ( - validation.get("site", {}).get("found") - and validation.get("device_type", {}).get("found") - and validation.get("device_role", {}).get("found") + is_ready = ( + can_import + and bool(validation.get("site", {}).get("found")) + and bool(validation.get("device_type", {}).get("found")) + and bool(validation.get("device_role", {}).get("found")) ) - validation["can_import"] = validation["is_ready"] = bool(required_found and not validation.get("issues")) + validation["can_import"] = can_import + validation["is_ready"] = is_ready except Exception as e: existing_id = getattr(existing, "pk", "unknown") if existing else "none" logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") From 3ac5a36cef0d47cc4b677af73d84fb495bfccb83 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 10:13:55 +0100 Subject: [PATCH 15/27] fix: batch 4 PR review fixes - virtual_chassis.py: load VC member name pattern once before loops to avoid N DB queries per member creation/update - virtual_chassis.py: fix parent_index check (0 is valid, use 'is None') - actions.py: use _resolve_naming_preferences() in link/update/sync_name actions instead of reading POST toggles directly (respects user prefs) - actions.py: HTML-escape incoming_serial and conflict_device.name in all serial-conflict HttpResponse messages (XSS fix) - actions.py: verify existing_device_id matches validated conflict target before mutation (only enforced when validation has existing_device set) - actions.py: add comment explaining collision check is functionally correct - bulk_import.py: remove dead wsgi.input try/except (no-op, cannot catch BrokenPipeError as written) - cache.py: fix filter_parts to use 'if v is not None' (preserves 0/False) - cache.py: normalize naive datetime to UTC before age_seconds subtraction - cache.py: update docstring example to reflect SHA-256 hex prefix format - filters.py: use api.server_key (always resolved) in cache key construction - device_operations.py: add comments explaining direct CF access is needed for legacy integer format detection (not a bug) - test_import_utils.py: add comment explaining dcim.models.Platform patch is correct (inline import at call time) - test_import_utils.py: fix _create_request to always include both toggle keys to prevent DB access in _resolve_naming_preferences fallback --- .../import_utils/bulk_import.py | 8 --- netbox_librenms_plugin/import_utils/cache.py | 10 ++- .../import_utils/device_operations.py | 10 ++- .../import_utils/filters.py | 5 +- .../import_utils/virtual_chassis.py | 41 ++++++++---- .../tests/test_import_utils.py | 16 +++-- .../views/imports/actions.py | 66 +++++++++---------- 7 files changed, 88 insertions(+), 68 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 848cc03d05..71d2176a2a 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -440,14 +440,6 @@ def process_device_filters( if job.job.status == JobStatusChoices.STATUS_FAILED: job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") return ([], False) if return_cache_status else [] - elif request: - # Check for client disconnect - try: - if hasattr(request, "META") and request.META.get("wsgi.input"): - pass - except (BrokenPipeError, ConnectionError, IOError): - logger.info(f"Client disconnected during validation at device {idx}") - return ([], False) if return_cache_status else [] # Drop any cached validation/meta keys before recomputing device.pop("_validation", None) diff --git a/netbox_librenms_plugin/import_utils/cache.py b/netbox_librenms_plugin/import_utils/cache.py index 7430d129f4..03f711ad02 100644 --- a/netbox_librenms_plugin/import_utils/cache.py +++ b/netbox_librenms_plugin/import_utils/cache.py @@ -21,8 +21,9 @@ def get_cache_metadata_key(server_key: str, filters: dict, vc_enabled: bool) -> Returns: str: Consistent cache key for metadata """ - # Sort filter items to ensure consistent key generation - filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v) + # Sort filter items to ensure consistent key generation; use "is not None" to preserve + # valid falsy values like 0 and False (filtering only None/missing entries). + filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v is not None) return f"librenms_filter_cache_metadata_{server_key}_{filter_parts}_{vc_enabled}" @@ -80,6 +81,9 @@ def get_active_cached_searches(server_key: str) -> list[dict]: cached_at = ( datetime.fromisoformat(cached_at_raw) if cached_at_raw else datetime.fromtimestamp(0, timezone.utc) ) + # Normalize naive datetimes (e.g., stored without tzinfo) to UTC + if cached_at.tzinfo is None: + cached_at = cached_at.replace(tzinfo=timezone.utc) except (ValueError, TypeError): cached_at = datetime.fromtimestamp(0, timezone.utc) age_seconds = (now - cached_at).total_seconds() @@ -136,7 +140,7 @@ def get_validated_device_cache_key(server_key: str, filters: dict, device_id: in Example: >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) >>> key - 'validated_device_default_-1234567890_123_vc' + 'validated_device_default_-e3b0c44298fc1c14_123_vc' """ # Sort filters for a deterministic, cross-process stable hash filter_hash = hashlib.sha256(json.dumps(sorted(filters.items()), sort_keys=True).encode()).hexdigest()[:16] diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index 8113c31fad..f7dbb7e1bc 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -234,7 +234,10 @@ def validate_device_for_import( result["import_as_vm"] = True # Force VM mode since VM exists result["can_import"] = False - # Detect legacy bare-integer format so UI can offer a migration action + # Detect legacy bare-integer format so UI can offer a migration action. + # Direct access needed to detect legacy integer format for migration prompt: + # LibreNMSAPI.get_librenms_id() returns an int in both formats, so only the + # raw type check on custom_field_data reveals whether migration is needed. if isinstance(existing_vm.custom_field_data.get("librenms_id"), int): result["librenms_id_needs_migration"] = True @@ -261,7 +264,10 @@ def validate_device_for_import( result["existing_match_type"] = "librenms_id" result["can_import"] = False - # Detect legacy bare-integer format so UI can offer a migration action + # Detect legacy bare-integer format so UI can offer a migration action. + # Direct access needed to detect legacy integer format for migration prompt: + # LibreNMSAPI.get_librenms_id() returns an int in both formats, so only the + # raw type check on custom_field_data reveals whether migration is needed. if isinstance(existing_device.custom_field_data.get("librenms_id"), int): result["librenms_id_needs_migration"] = True diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py index 9f55576a26..4085dfa325 100644 --- a/netbox_librenms_plugin/import_utils/filters.py +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -172,13 +172,14 @@ def get_librenms_devices_for_import( # We'll filter client-side if needed # Use caching to avoid repeated API calls - # Include both API and client filters in cache key (deterministic, cross-process stable) + # Include both API and client filters in cache key (deterministic, cross-process stable). + # Use api.server_key (always resolved) rather than the raw server_key arg (may differ). def _hash(d): return hashlib.sha256( json.dumps(sorted(d.items()) if isinstance(d, dict) else d, sort_keys=True).encode() ).hexdigest()[:16] - cache_key = f"librenms_devices_import_{server_key}_{_hash(api_filters)}_{_hash(client_filters)}" + cache_key = f"librenms_devices_import_{api.server_key}_{_hash(api_filters)}_{_hash(client_filters)}" from_cache = False if force_refresh: diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py index 4dc44e4182..7f775faa79 100644 --- a/netbox_librenms_plugin/import_utils/virtual_chassis.py +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -175,7 +175,7 @@ def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> d logger.debug(f"VC detection: Found parent container at index {parent_index} for device {device_id}") break - if not parent_index: + if parent_index is None: return None # Step 3: Get children chassis at next level @@ -232,7 +232,19 @@ def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> d return None -def _generate_vc_member_name(master_name: str, position: int, serial: str = None) -> str: +def _load_vc_member_name_pattern() -> str: + """Load the VC member name pattern from settings, with fallback to default.""" + from ..models import LibreNMSSettings + + try: + settings = LibreNMSSettings.objects.first() + return settings.vc_member_name_pattern if settings else "-M{position}" + except Exception as e: + logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") + return "-M{position}" + + +def _generate_vc_member_name(master_name: str, position: int, serial: str = None, pattern: str = None) -> str: """ Generate name for VC member device using configured pattern from settings. @@ -240,6 +252,9 @@ def _generate_vc_member_name(master_name: str, position: int, serial: str = None master_name: Name of the master/primary device position: VC position number serial: Optional serial number of the member device + pattern: Optional pre-loaded name pattern; if None, loaded from settings. + Pass a pre-loaded pattern when calling inside a loop to avoid + repeated DB queries. Returns: Generated member device name @@ -250,16 +265,8 @@ def _generate_vc_member_name(master_name: str, position: int, serial: str = None pattern="-SW{position}" -> "switch01-SW2" pattern=" [{serial}]" -> "switch01 [ABC123]" """ - # Import here to avoid circular dependency - from ..models import LibreNMSSettings - - # Get pattern from settings with fallback to default - try: - settings = LibreNMSSettings.objects.first() - pattern = settings.vc_member_name_pattern if settings else "-M{position}" - except Exception as e: - logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") - pattern = "-M{position}" + if pattern is None: + pattern = _load_vc_member_name_pattern() # Prepare format variables format_vars = { @@ -294,6 +301,8 @@ def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: if not vc_data or not vc_data.get("is_stack"): return vc_data + # Load naming pattern once to avoid a DB query per member + vc_pattern = _load_vc_member_name_pattern() for idx, member in enumerate(vc_data.get("members", [])): raw_position = member.get("position", idx) try: @@ -302,7 +311,9 @@ def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: base_position = idx position = base_position + 1 # Convert to 1-based position member["position"] = base_position - member["suggested_name"] = _generate_vc_member_name(master_name, position, serial=member.get("serial")) + member["suggested_name"] = _generate_vc_member_name( + master_name, position, serial=member.get("serial"), pattern=vc_pattern + ) return vc_data @@ -371,6 +382,8 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis # Create member devices for remaining positions position = 2 # Start at 2 (master is 1) members_created = 0 + # Load naming pattern once to avoid a DB query per member + vc_pattern = _load_vc_member_name_pattern() for member in members_info: # Skip if this is the master's serial (only when both serials are non-empty) @@ -389,7 +402,7 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") continue - member_name = _generate_vc_member_name(master_base_name, position, serial=serial) + member_name = _generate_vc_member_name(master_base_name, position, serial=serial, pattern=vc_pattern) # Check for duplicate name if Device.objects.filter(name=member_name).exists(): diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 27c157b1dd..ba88025fc3 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1759,11 +1759,14 @@ def _create_view(self): def _create_request(self, action, existing_device_id, use_sysname=False, strip_domain=False): """Create a mock request with POST data.""" request = MagicMock() - post_data = {"action": action, "existing_device_id": str(existing_device_id)} - if use_sysname: - post_data["use-sysname-toggle"] = "on" - if strip_domain: - post_data["strip-domain-toggle"] = "on" + # Always include both toggles so _resolve_naming_preferences never falls through + # to the user-pref/settings DB path, which would hit the real database. + post_data = { + "action": action, + "existing_device_id": str(existing_device_id), + "use-sysname-toggle": "on" if use_sysname else "off", + "strip-domain-toggle": "on" if strip_domain else "off", + } request.POST = post_data return request @@ -2199,6 +2202,9 @@ def test_sync_platform_action(self, mock_cache_key, mock_cache): patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + # Patch at dcim.models level: find_matching_platform uses an inline + # 'from dcim.models import Platform' so patching dcim.models.Platform + # correctly intercepts the binding at call time. patch("dcim.models.Platform") as mock_platform_cls, ): mock_device_cls.objects.get.return_value = existing_device diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 21682e6403..41e19b669f 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -880,6 +880,13 @@ def post(self, request, device_id): if not libre_device: return HttpResponse("LibreNMS device not found", status=404) + # Verify the POSTed existing_device_id matches the validated conflict target. + # Without this check, an attacker could mutate an arbitrary device. + # Only enforce when validation has a known existing_device (conflict case). + validated_existing = validation.get("existing_device") if validation else None + if validated_existing is not None and validated_existing.pk != existing_device.pk: + return HttpResponse("Device ID mismatch: existing_device_id does not match validated device", status=400) + # Require force flag when device type mismatches, but only for actions that use it _FORCE_REQUIRED_ACTIONS = {"link", "update", "update_serial", "update_type"} force = request.POST.get("force") == "on" @@ -900,7 +907,10 @@ def post(self, request, device_id): except (TypeError, ValueError): return HttpResponse("Invalid or missing LibreNMS device_id in payload", status=400) - # Check for LibreNMS ID collision before any linking action + # Check for LibreNMS ID collision before any linking action. + # find_by_librenms_id returns None or the *one* device matching the ID. + # Comparing .pk != existing_device.pk is equivalent to .exclude(pk=...).exists() + # for conflict detection — both find any *other* device with the same librenms_id. if action in {"link", "update", "update_serial"}: from netbox_librenms_plugin.utils import find_by_librenms_id @@ -915,15 +925,11 @@ def post(self, request, device_id): if action == "link": # Link to LibreNMS and update name from LibreNMS data resolved_name = validation.get("resolved_name") - hostname = ( - resolved_name - if resolved_name - else _determine_device_name( - libre_device, - use_sysname=request.POST.get("use-sysname-toggle") == "on", - strip_domain=request.POST.get("strip-domain-toggle") == "on", - ) - ) + if resolved_name: + hostname = resolved_name + else: + use_sysname, strip_domain = _resolve_naming_preferences(request) + hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) existing_device.name = hostname if librenms_device_type: @@ -935,21 +941,17 @@ def post(self, request, device_id): # Update hostname, serial, and link to LibreNMS resolved_name = validation.get("resolved_name") incoming_serial = libre_device.get("serial") or "" - hostname = ( - resolved_name - if resolved_name - else _determine_device_name( - libre_device, - use_sysname=request.POST.get("use-sysname-toggle") == "on", - strip_domain=request.POST.get("strip-domain-toggle") == "on", - ) - ) + if resolved_name: + hostname = resolved_name + else: + use_sysname, strip_domain = _resolve_naming_preferences(request) + hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) if incoming_serial and incoming_serial != "-": conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() if conflict_device: return HttpResponse( - f"Serial conflict: '{incoming_serial}' is already assigned to device " - f"'{conflict_device.name}' (ID: {conflict_device.pk})", + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", status=409, ) existing_device.serial = incoming_serial @@ -970,8 +972,8 @@ def post(self, request, device_id): conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() if conflict_device: return HttpResponse( - f"Serial conflict: '{incoming_serial}' is already assigned to device " - f"'{conflict_device.name}' (ID: {conflict_device.pk})", + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", status=409, ) existing_device.serial = incoming_serial @@ -987,15 +989,11 @@ def post(self, request, device_id): elif action == "sync_name": # Sync device name from LibreNMS (e.g., IP → sysName) resolved_name = validation.get("resolved_name") - hostname = ( - resolved_name - if resolved_name - else _determine_device_name( - libre_device, - use_sysname=request.POST.get("use-sysname-toggle") == "on", - strip_domain=request.POST.get("strip-domain-toggle") == "on", - ) - ) + if resolved_name: + hostname = resolved_name + else: + use_sysname, strip_domain = _resolve_naming_preferences(request) + hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) existing_device.name = hostname existing_device.save() logger.info(f"Synced name on device '{existing_device.name}' from LibreNMS") @@ -1021,8 +1019,8 @@ def post(self, request, device_id): f"'{conflict_device.name}' (pk={conflict_device.pk})" ) return HttpResponse( - f"Serial conflict: '{incoming_serial}' is already assigned to device " - f"'{conflict_device.name}' (ID: {conflict_device.pk})", + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", status=409, ) existing_device.serial = incoming_serial From 0b9ae51fc856817a3a2a5174baa8b2492d00bd10 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 11:24:02 +0100 Subject: [PATCH 16/27] fix: batch 5 PR review fixes - bulk_import.py: add _empty_return() helper, replace 5 inline '([], False) if return_cache_status else []' with calls to it - cache.py: fix docstring example (spurious '-' before hash digest) - filters.py: normalize status filter value to int before comparison so string '1'/'0' from form fields is handled correctly - virtual_chassis.py: preserve discovered SNMP vc_position when available; fall back to sequential counter only when position is absent - actions.py: require validated_existing to be non-None for conflict actions (missing target now returns 400 instead of proceeding) - actions.py: replace find_by_librenms_id collision check with an exclusion-aware queryset (.exclude(pk=).exists()) for cleaner intent - actions.py: verify legacy cf_value matches librenms_id before migration to prevent migrating stale/incorrect associations - test_import_utils.py: add existing_device to all conflict-action test validation dicts to match the new required-target enforcement - test_import_utils.py: patch find_matching_platform at utility module level instead of dcim.models.Platform --- .../import_utils/bulk_import.py | 17 +++++--- netbox_librenms_plugin/import_utils/cache.py | 2 +- .../import_utils/filters.py | 9 +++- .../import_utils/virtual_chassis.py | 15 +++++-- .../tests/test_import_utils.py | 31 ++++++++------ .../views/imports/actions.py | 42 +++++++++++++------ 6 files changed, 80 insertions(+), 36 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 71d2176a2a..5ddeceb723 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -20,6 +20,11 @@ logger = logging.getLogger(__name__) +def _empty_return(return_cache_status: bool): + """Centralised empty-result return value for process_device_filters.""" + return ([], False) if return_cache_status else [] + + def bulk_import_devices_shared( device_ids: List[int], server_key: str = None, @@ -384,7 +389,7 @@ def process_device_filters( except (BrokenPipeError, ConnectionError, IOError) as e: if request: logger.info(f"Client disconnected during VC prefetch: {e}") - return ([], False) if return_cache_status else [] + return _empty_return(return_cache_status) raise # Validate each device @@ -404,13 +409,13 @@ def process_device_filters( if rq_job.is_failed or rq_job.is_stopped: job.logger.warning("Job was already stopped before validation started") - return ([], False) if return_cache_status else [] + return _empty_return(return_cache_status) except Exception: # Fall back to DB check if RQ check fails job.job.refresh_from_db() if job.job.status == JobStatusChoices.STATUS_FAILED: job.logger.warning("Job was stopped before validation started") - return ([], False) if return_cache_status else [] + return _empty_return(return_cache_status) else: logger.info(f"Validating {total} devices") @@ -433,13 +438,13 @@ def process_device_filters( job.logger.info( f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." ) - return ([], False) if return_cache_status else [] + return _empty_return(return_cache_status) except Exception: # If we can't check RQ status, fall back to DB status check job.job.refresh_from_db() if job.job.status == JobStatusChoices.STATUS_FAILED: job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") - return ([], False) if return_cache_status else [] + return _empty_return(return_cache_status) # Drop any cached validation/meta keys before recomputing device.pop("_validation", None) @@ -486,7 +491,7 @@ def process_device_filters( except (BrokenPipeError, ConnectionError, IOError) as e: if request: logger.info(f"Client disconnected during device validation: {e}") - return ([], False) if return_cache_status else [] + return _empty_return(return_cache_status) raise # Set VC detection metadata diff --git a/netbox_librenms_plugin/import_utils/cache.py b/netbox_librenms_plugin/import_utils/cache.py index 03f711ad02..627080b506 100644 --- a/netbox_librenms_plugin/import_utils/cache.py +++ b/netbox_librenms_plugin/import_utils/cache.py @@ -140,7 +140,7 @@ def get_validated_device_cache_key(server_key: str, filters: dict, device_id: in Example: >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) >>> key - 'validated_device_default_-e3b0c44298fc1c14_123_vc' + 'validated_device_default_e3b0c44298fc1c14_123_vc' """ # Sort filters for a deterministic, cross-process stable hash filter_hash = hashlib.sha256(json.dumps(sorted(filters.items()), sort_keys=True).encode()).hexdigest()[:16] diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py index 4085dfa325..f312f299c2 100644 --- a/netbox_librenms_plugin/import_utils/filters.py +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -87,10 +87,15 @@ def get_librenms_devices_for_import( if filters: # Check for status filter first - it has special handling if filters.get("status") is not None: + # Normalize to int: form fields send strings ("1"/"0"), API may send ints + try: + status_val = int(filters["status"]) + except (ValueError, TypeError): + status_val = None # Status filter uses special types that don't need query param - if filters["status"] == 1: + if status_val == 1: api_filters["type"] = "up" - elif filters["status"] == 0: + elif status_val == 0: api_filters["type"] = "down" # Save ALL other filters for client-side filtering when status is used diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py index 7f775faa79..70778098cc 100644 --- a/netbox_librenms_plugin/import_utils/virtual_chassis.py +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -402,7 +402,17 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") continue - member_name = _generate_vc_member_name(master_base_name, position, serial=serial, pattern=vc_pattern) + # Prefer the discovered SNMP position; fall back to sequential counter + try: + discovered_pos = int(member.get("position")) if member.get("position") is not None else None + except (TypeError, ValueError): + discovered_pos = None + chosen_pos = discovered_pos if discovered_pos is not None else position + # Advance sequential counter only when it was consumed as a fallback + if discovered_pos is None: + position += 1 + + member_name = _generate_vc_member_name(master_base_name, chosen_pos, serial=serial, pattern=vc_pattern) # Check for duplicate name if Device.objects.filter(name=member_name).exists(): @@ -419,12 +429,11 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis platform=master_device.platform, serial=serial, virtual_chassis=vc, - vc_position=position, + vc_position=chosen_pos, comments=f"VC member (LibreNMS: {member.get('name', 'Unknown')})\n" f"Auto-created from stack inventory", ) members_created += 1 - position += 1 # Validate member count expected_members = len( diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index ba88025fc3..b9d358762e 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1788,7 +1788,7 @@ def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache) "sysName": "switch-01.example.com", "serial": "ABC123", } - validation = {"can_import": False} + validation = {"can_import": False, "existing_device": existing_device} selections = {} request = self._create_request("link", 42, use_sysname=True) @@ -1801,6 +1801,7 @@ def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache) mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1829,7 +1830,7 @@ def test_update_action_sets_hostname_serial_and_librenms_id(self, mock_cache_key "sysName": "new-name.example.com", "serial": "NEW-SERIAL", } - validation = {"can_import": False} + validation = {"can_import": False, "existing_device": existing_device} selections = {} request = self._create_request("update", 42, use_sysname=True) @@ -1842,6 +1843,7 @@ def test_update_action_sets_hostname_serial_and_librenms_id(self, mock_cache_key mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1866,7 +1868,7 @@ def test_update_serial_action_updates_serial_only(self, mock_cache_key, mock_cac existing_device.serial = "OLD-SERIAL" libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "NEW-SERIAL"} - validation = {"can_import": False} + validation = {"can_import": False, "existing_device": existing_device} selections = {} request = self._create_request("update_serial", 42) @@ -1879,6 +1881,7 @@ def test_update_serial_action_updates_serial_only(self, mock_cache_key, mock_cac mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1904,7 +1907,7 @@ def test_update_skips_dash_serial(self, mock_cache_key, mock_cache): existing_device.serial = "EXISTING" libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "-"} - validation = {"can_import": False} + validation = {"can_import": False, "existing_device": existing_device} selections = {} request = self._create_request("update_serial", 42) @@ -1971,7 +1974,7 @@ def test_sync_name_action_updates_name(self, mock_cache_key, mock_cache): "sysName": "switch-01.example.com", "serial": "ABC123", } - validation = {"can_import": False} + validation = {"can_import": False, "existing_device": existing_device} selections = {} request = self._create_request("sync_name", 42, use_sysname=True) @@ -2002,7 +2005,7 @@ def test_device_type_mismatch_blocked_without_force(self, mock_cache_key, mock_c existing_device.custom_field_data = {} libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "ABC123"} - validation = {"can_import": False, "device_type_mismatch": True} + validation = {"can_import": False, "device_type_mismatch": True, "existing_device": existing_device} selections = {} request = self._create_request("link", 42, use_sysname=True) @@ -2037,7 +2040,7 @@ def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cach "sysName": "switch-01.example.com", "serial": "ABC123", } - validation = {"can_import": False, "device_type_mismatch": True} + validation = {"can_import": False, "device_type_mismatch": True, "existing_device": existing_device} selections = {} request = self._create_request("link", 42, use_sysname=True) @@ -2051,6 +2054,7 @@ def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cach mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -2083,6 +2087,7 @@ def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cach "can_import": False, "device_type_mismatch": True, "device_type": {"device_type": librenms_device_type}, + "existing_device": existing_device, } selections = {} @@ -2097,6 +2102,7 @@ def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cach mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -2131,6 +2137,7 @@ def test_update_type_action_changes_device_type(self, mock_cache_key, mock_cache "can_import": False, "device_type_mismatch": True, "device_type": {"device_type": new_device_type}, + "existing_device": existing_device, } selections = {} @@ -2174,6 +2181,7 @@ def test_sync_serial_action(self, mock_cache_key, mock_cache): mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -2202,13 +2210,12 @@ def test_sync_platform_action(self, mock_cache_key, mock_cache): patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, - # Patch at dcim.models level: find_matching_platform uses an inline - # 'from dcim.models import Platform' so patching dcim.models.Platform - # correctly intercepts the binding at call time. - patch("dcim.models.Platform") as mock_platform_cls, + # Patch find_matching_platform at the utility module level — the action imports + # it from netbox_librenms_plugin.utils, so that is the correct seam to mock. + patch("netbox_librenms_plugin.utils.find_matching_platform") as mock_find_platform, ): mock_device_cls.objects.get.return_value = existing_device - mock_platform_cls.objects.get.return_value = mock_platform + mock_find_platform.return_value = {"found": True, "platform": mock_platform, "match_type": "exact"} mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 41e19b669f..e98bbbe539 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -881,10 +881,12 @@ def post(self, request, device_id): return HttpResponse("LibreNMS device not found", status=404) # Verify the POSTed existing_device_id matches the validated conflict target. - # Without this check, an attacker could mutate an arbitrary device. - # Only enforce when validation has a known existing_device (conflict case). + # Require a confirmed conflict target: if validation has no existing_device, the + # LibreNMS device was not validated against this NetBox device, so mutations are unsafe. validated_existing = validation.get("existing_device") if validation else None - if validated_existing is not None and validated_existing.pk != existing_device.pk: + if validated_existing is None: + return HttpResponse("Missing validated conflict target", status=400) + if validated_existing.pk != existing_device.pk: return HttpResponse("Device ID mismatch: existing_device_id does not match validated device", status=400) # Require force flag when device type mismatches, but only for actions that use it @@ -907,18 +909,23 @@ def post(self, request, device_id): except (TypeError, ValueError): return HttpResponse("Invalid or missing LibreNMS device_id in payload", status=400) - # Check for LibreNMS ID collision before any linking action. - # find_by_librenms_id returns None or the *one* device matching the ID. - # Comparing .pk != existing_device.pk is equivalent to .exclude(pk=...).exists() - # for conflict detection — both find any *other* device with the same librenms_id. + # Check for LibreNMS ID collision before any linking action: detect any *other* + # Device that already carries the same librenms_id, excluding the current target. if action in {"link", "update", "update_serial"}: - from netbox_librenms_plugin.utils import find_by_librenms_id + from django.db.models import Q - id_conflict = find_by_librenms_id(Device, librenms_id, self.librenms_api.server_key) - if id_conflict and id_conflict.pk != existing_device.pk: + server_key = self.librenms_api.server_key + conflict_exists = ( + Device.objects.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}) + | Q(custom_field_data__librenms_id=librenms_id) + ) + .exclude(pk=existing_device.pk) + .exists() + ) + if conflict_exists: return HttpResponse( - f"LibreNMS ID conflict: ID {librenms_id} is already assigned to device " - f"'{escape(id_conflict.name)}' (ID: {id_conflict.pk})", + f"LibreNMS ID conflict: ID {librenms_id} is already assigned to another device.", status=409, ) @@ -1064,12 +1071,23 @@ def post(self, request, device_id): # confirmed by serial match (or explicit force). from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + # Direct access needed to detect legacy integer format for migration prompt: + # LibreNMSAPI.get_librenms_id() returns an int in both formats; only the raw + # type check on custom_field_data reveals whether migration is needed. cf_value = existing_device.custom_field_data.get("librenms_id") if not isinstance(cf_value, int): return HttpResponse( "Device librenms_id is already in JSON format; no migration needed.", status=400, ) + # Verify the stored legacy ID matches the active LibreNMS device_id so we don't + # migrate a stale/incorrect association to the wrong server mapping. + if cf_value != librenms_id: + return HttpResponse( + f"Legacy librenms_id ({cf_value}) does not match the active device ID " + f"({librenms_id}); cannot migrate safely.", + status=400, + ) if not validation.get("serial_confirmed") and not force: return HttpResponse( "Serial number not confirmed. Check the force checkbox to migrate without serial verification.", From 022293b999425dd2e5274d94983eebead68bb347 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 14:01:26 +0100 Subject: [PATCH 17/27] fix: batch 6 PR review fixes - bulk_import.py: normalize status (int/str) in show_disabled filter - bulk_import.py: use api.server_key (resolved) in import_single_device call - filters.py: normalize status in count_librenms_devices show_disabled filter - virtual_chassis.py: fix 0-based fallback for vc_position (idx+1 not idx) - virtual_chassis.py: advance sequential counter past discovered_pos; normalize pos=0 to absent - actions.py: add django.db.transaction import - actions.py: module-level _FORCE_REQUIRED_ACTIONS frozenset - actions.py: defensive .get('resolved_name') instead of [] access - actions.py: add NetBoxObjectPermissionMixin to DeviceConflictActionView - actions.py: wrap librenms_id collision check+write in transaction.atomic() - tests: fix unknown-action test to include existing_device in validation dict - tests: update warning text assertion to 'hostname differs' - tests: add transaction mock to 6 link/update/update_serial tests - tests: add TestProcessDeviceFilters with 5 new tests for status norm + api.server_key --- .../import_utils/bulk_import.py | 14 +- .../import_utils/filters.py | 12 +- .../import_utils/virtual_chassis.py | 17 +- .../tests/test_import_utils.py | 186 +++++++++++++++++- .../views/imports/actions.py | 175 ++++++++-------- 5 files changed, 315 insertions(+), 89 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 5ddeceb723..b515a46045 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -154,7 +154,7 @@ def bulk_import_devices_shared( result = import_single_device( device_id, - server_key=server_key, + server_key=api.server_key, # use resolved key, not raw parameter (may be None) sync_options=sync_options, manual_mappings=device_mappings if device_mappings else None, libre_device=libre_device, @@ -363,9 +363,17 @@ def process_device_filters( return_cache_status=True, ) - # Filter out disabled devices if requested + # Filter out disabled devices if requested; normalize status to int to handle + # both integer (1) and string ("1") responses from the LibreNMS API. if not show_disabled: - libre_devices = [d for d in libre_devices if d.get("status") == 1] + + def _is_active(d): + try: + return int(d.get("status", 0)) == 1 + except (TypeError, ValueError): + return False + + libre_devices = [d for d in libre_devices if _is_active(d)] if job: job.logger.info(f"Found {len(libre_devices)} devices to process") diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py index f312f299c2..a7890c9293 100644 --- a/netbox_librenms_plugin/import_utils/filters.py +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -35,9 +35,17 @@ def get_device_count_for_filters( """ devices = get_librenms_devices_for_import(api, filters=filters, force_refresh=clear_cache) - # Filter out disabled devices if requested + # Filter out disabled devices if requested; normalize status to int to handle + # both integer (1) and string ("1") responses from the LibreNMS API. if not show_disabled: - devices = [d for d in devices if d.get("status") == 1] + + def _is_active(d): + try: + return int(d.get("status", 0)) == 1 + except (TypeError, ValueError): + return False + + devices = [d for d in devices if _is_active(d)] return len(devices) diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py index 70778098cc..2f030875e4 100644 --- a/netbox_librenms_plugin/import_utils/virtual_chassis.py +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -198,11 +198,13 @@ def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> d # Step 5: Extract member info members = [] for idx, chassis in enumerate(chassis_items): - raw_position = chassis.get("entPhysicalParentRelPos", idx) + # entPhysicalParentRelPos is 1-based; fall back to idx+1 (not idx) so + # position 0 is never produced — VC positions must be ≥ 1. + raw_position = chassis.get("entPhysicalParentRelPos", idx + 1) try: position = int(raw_position) except (TypeError, ValueError): - position = idx + position = idx + 1 member_data = { "serial": chassis.get("entPhysicalSerialNum", ""), "position": position, @@ -402,15 +404,22 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") continue - # Prefer the discovered SNMP position; fall back to sequential counter + # Prefer the discovered SNMP position; fall back to sequential counter. + # Normalize discovered_pos: 0 is not a valid VC position, treat as absent. try: discovered_pos = int(member.get("position")) if member.get("position") is not None else None except (TypeError, ValueError): discovered_pos = None + if discovered_pos is not None and discovered_pos < 1: + discovered_pos = None # 0 is invalid for vc_position; fall back to counter chosen_pos = discovered_pos if discovered_pos is not None else position - # Advance sequential counter only when it was consumed as a fallback + # Advance the sequential counter: + # - if discovered_pos was used, advance counter past it to avoid future reuse; + # - if counter was consumed as fallback, increment it normally. if discovered_pos is None: position += 1 + else: + position = max(position, discovered_pos + 1) member_name = _generate_vc_member_name(master_base_name, chosen_pos, serial=serial, pattern=vc_pattern) diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index b9d358762e..39ea7e72a1 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -1215,7 +1215,7 @@ def device_filter(**kwargs): assert result["serial_action"] == "hostname_differs" assert result["existing_match_type"] == "serial" - assert "reinstalled" in result["warnings"][0] + assert "hostname differs" in result["warnings"][0] def test_hostname_match_diff_serial_offers_update(self): """Hostname matches but serial differs offers update_serial action.""" @@ -1797,7 +1797,9 @@ def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache) patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None @@ -1839,7 +1841,9 @@ def test_update_action_sets_hostname_serial_and_librenms_id(self, mock_cache_key patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None @@ -1877,7 +1881,9 @@ def test_update_serial_action_updates_serial_only(self, mock_cache_key, mock_cac patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None @@ -1916,8 +1922,11 @@ def test_update_skips_dash_serial(self, mock_cache_key, mock_cache): patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -1943,14 +1952,18 @@ def test_unknown_action_returns_400(self): request = self._create_request("invalid_action", 42) existing_device = MagicMock() + existing_device.pk = 42 libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "ABC"} with ( patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "require_object_permissions", return_value=None), patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device - mock_validate.return_value = (libre_device, {}, {}) + # Include existing_device so the validated-conflict-target guard passes; + # we want to exercise the unknown-action branch, not the missing-device guard. + mock_validate.return_value = (libre_device, {"existing_device": existing_device}, {}) response = view.post(request, device_id=10) @@ -2050,7 +2063,9 @@ def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cach patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None @@ -2098,7 +2113,9 @@ def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cach patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None @@ -2529,3 +2546,168 @@ def test_backward_compatible_defaults(self, *mocks): result = validate_device_for_import(device_data, include_vc_detection=False) assert "resolved_name" in result assert result["resolved_name"] == "switch-01" + + +class TestProcessDeviceFilters: + """Tests for process_device_filters and related bulk_import utilities.""" + + def test_show_disabled_filters_integer_status_1(self): + """show_disabled=False should keep devices with status==1 (int).""" + from unittest.mock import MagicMock, patch + + devices = [ + {"device_id": 1, "hostname": "a", "status": 1}, + {"device_id": 2, "hostname": "b", "status": 0}, + ] + with ( + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_librenms_devices_for_import", + return_value=(devices, False), + ), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import", + side_effect=lambda d, **kw: { + "resolved_name": d["hostname"], + "is_ready": True, + "can_import": True, + "status": "active", + "existing_device": None, + "import_as_vm": False, + "existing_match_type": None, + }, + ), + patch("netbox_librenms_plugin.import_utils.bulk_import.prefetch_vc_data_for_devices"), + patch("netbox_librenms_plugin.import_utils.bulk_import.cache"), + patch("netbox_librenms_plugin.import_utils.bulk_import.get_cache_metadata_key", return_value="key"), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_validated_device_cache_key", return_value="vkey" + ), + ): + from netbox_librenms_plugin.import_utils.bulk_import import process_device_filters + + api = MagicMock() + api.server_key = "default" + result = process_device_filters( + api, filters={}, vc_detection_enabled=False, clear_cache=False, show_disabled=False + ) + + # Only device with status==1 should be processed; disabled device excluded before validation + assert len(result) == 1 + assert result[0]["hostname"] == "a" + + def test_show_disabled_filters_string_status_1(self): + """show_disabled=False should keep devices with status=='1' (string from API).""" + from unittest.mock import MagicMock, patch + + devices = [ + {"device_id": 1, "hostname": "a", "status": "1"}, + {"device_id": 2, "hostname": "b", "status": "0"}, + ] + with ( + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_librenms_devices_for_import", + return_value=(devices, False), + ), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import", + side_effect=lambda d, **kw: { + "resolved_name": d["hostname"], + "is_ready": True, + "can_import": True, + "status": "active", + "existing_device": None, + "import_as_vm": False, + "existing_match_type": None, + }, + ), + patch("netbox_librenms_plugin.import_utils.bulk_import.prefetch_vc_data_for_devices"), + patch("netbox_librenms_plugin.import_utils.bulk_import.cache"), + patch("netbox_librenms_plugin.import_utils.bulk_import.get_cache_metadata_key", return_value="key"), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_validated_device_cache_key", return_value="vkey" + ), + ): + from netbox_librenms_plugin.import_utils.bulk_import import process_device_filters + + api = MagicMock() + api.server_key = "default" + result = process_device_filters( + api, filters={}, vc_detection_enabled=False, clear_cache=False, show_disabled=False + ) + + assert len(result) == 1 + assert result[0]["hostname"] == "a" + + def test_show_disabled_true_includes_all(self): + """show_disabled=True should include both active and inactive devices.""" + from unittest.mock import MagicMock, patch + + devices = [ + {"device_id": 1, "hostname": "a", "status": 1}, + {"device_id": 2, "hostname": "b", "status": 0}, + ] + with ( + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_librenms_devices_for_import", + return_value=(devices, False), + ), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import", + side_effect=lambda d, **kw: { + "resolved_name": d["hostname"], + "is_ready": True, + "can_import": True, + "status": "active", + "existing_device": None, + "import_as_vm": False, + "existing_match_type": None, + }, + ), + patch("netbox_librenms_plugin.import_utils.bulk_import.prefetch_vc_data_for_devices"), + patch("netbox_librenms_plugin.import_utils.bulk_import.cache"), + patch("netbox_librenms_plugin.import_utils.bulk_import.get_cache_metadata_key", return_value="key"), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_validated_device_cache_key", return_value="vkey" + ), + ): + from netbox_librenms_plugin.import_utils.bulk_import import process_device_filters + + api = MagicMock() + api.server_key = "default" + result = process_device_filters( + api, filters={}, vc_detection_enabled=False, clear_cache=False, show_disabled=True + ) + + assert len(result) == 2 + + def test_empty_return_helper(self): + """_empty_return should return ([], False) when return_cache_status=True, else [].""" + from netbox_librenms_plugin.import_utils.bulk_import import _empty_return + + assert _empty_return(True) == ([], False) + assert _empty_return(False) == [] + + def test_bulk_import_devices_uses_resolved_server_key(self): + """bulk_import_devices_shared should pass api.server_key to import_single_device.""" + from unittest.mock import MagicMock, patch + + with ( + patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") as mock_api_cls, + patch("netbox_librenms_plugin.import_utils.bulk_import.import_single_device") as mock_import, + patch("netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import"), + patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions"), + ): + mock_api = MagicMock() + mock_api.server_key = "resolved-key" + mock_api.get_device_info.return_value = (True, {"device_id": 1, "hostname": "sw"}) + mock_api_cls.return_value = mock_api + mock_import.return_value = {"success": True, "device": MagicMock(), "is_vm": False} + + user = MagicMock() + from netbox_librenms_plugin.import_utils.bulk_import import bulk_import_devices_shared + + bulk_import_devices_shared([1], user=user, server_key=None) + + # The resolved api.server_key ("resolved-key") must be passed, not None + assert mock_import.call_args is not None + assert mock_import.call_args.kwargs.get("server_key") == "resolved-key" diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index e98bbbe539..ac724088d3 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -6,6 +6,7 @@ from django.contrib import messages from django.core.cache import cache from django.core.exceptions import PermissionDenied +from django.db import transaction from django.http import HttpResponse, JsonResponse from django.shortcuts import redirect, render from django.utils.html import escape @@ -31,10 +32,13 @@ ) from netbox_librenms_plugin.tables.device_status import DeviceImportTable from netbox_librenms_plugin.utils import get_user_pref, save_user_pref, set_librenms_device_id -from netbox_librenms_plugin.views.mixins import LibreNMSAPIMixin, LibreNMSPermissionMixin +from netbox_librenms_plugin.views.mixins import LibreNMSAPIMixin, LibreNMSPermissionMixin, NetBoxObjectPermissionMixin logger = logging.getLogger(__name__) +# Actions that require the force checkbox when a device-type mismatch is detected. +_FORCE_REQUIRED_ACTIONS = frozenset({"link", "update", "update_serial", "update_type"}) + def _resolve_naming_preferences(request) -> tuple[bool, bool]: """Resolve use_sysname/strip_domain: POST data → user pref → plugin settings.""" @@ -309,7 +313,7 @@ def post(self, request): vc_requested = request.GET.get("enable_vc_detection") == "true" validation["_vc_detection_enabled"] = vc_requested - device_name = validation["resolved_name"] + device_name = validation.get("resolved_name") if validation.get("virtual_chassis", {}).get("is_stack") and device_name: validation["virtual_chassis"] = update_vc_member_suggested_names( @@ -855,7 +859,9 @@ def post(self, request, device_id): return self.render_device_row(request, libre_device, validation, selections) -class DeviceConflictActionView(LibreNMSPermissionMixin, LibreNMSAPIMixin, DeviceImportHelperMixin, View): +class DeviceConflictActionView( + LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, LibreNMSAPIMixin, DeviceImportHelperMixin, View +): """HTMX view to resolve device conflicts (link, update, update serial).""" def post(self, request, device_id): @@ -876,6 +882,11 @@ def post(self, request, device_id): except (Device.DoesNotExist, ValueError): return HttpResponse("Existing device not found", status=404) + # Object-level change permission for the specific device being mutated. + self.required_object_permissions = {"POST": [("change", Device)]} + if error := self.require_object_permissions("POST"): + return error + libre_device, validation, selections = self.get_validated_device_with_selections(device_id, request) if not libre_device: return HttpResponse("LibreNMS device not found", status=404) @@ -890,7 +901,6 @@ def post(self, request, device_id): return HttpResponse("Device ID mismatch: existing_device_id does not match validated device", status=400) # Require force flag when device type mismatches, but only for actions that use it - _FORCE_REQUIRED_ACTIONS = {"link", "update", "update_serial", "update_type"} force = request.POST.get("force") == "on" if validation.get("device_type_mismatch") and action in _FORCE_REQUIRED_ACTIONS and not force: return HttpResponse( @@ -909,89 +919,98 @@ def post(self, request, device_id): except (TypeError, ValueError): return HttpResponse("Invalid or missing LibreNMS device_id in payload", status=400) - # Check for LibreNMS ID collision before any linking action: detect any *other* - # Device that already carries the same librenms_id, excluding the current target. + # Wrap the LibreNMS-ID collision check and subsequent write in a single + # transaction so the read-then-write is atomic for link/update/update_serial. if action in {"link", "update", "update_serial"}: from django.db.models import Q - server_key = self.librenms_api.server_key - conflict_exists = ( - Device.objects.filter( - Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}) - | Q(custom_field_data__librenms_id=librenms_id) - ) - .exclude(pk=existing_device.pk) - .exists() - ) - if conflict_exists: - return HttpResponse( - f"LibreNMS ID conflict: ID {librenms_id} is already assigned to another device.", - status=409, + with transaction.atomic(): + server_key = self.librenms_api.server_key + conflict_exists = ( + Device.objects.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}) + | Q(custom_field_data__librenms_id=librenms_id) + ) + .exclude(pk=existing_device.pk) + .exists() ) - - if action == "link": - # Link to LibreNMS and update name from LibreNMS data - resolved_name = validation.get("resolved_name") - if resolved_name: - hostname = resolved_name - else: - use_sysname, strip_domain = _resolve_naming_preferences(request) - hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) - set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) - existing_device.name = hostname - if librenms_device_type: - existing_device.device_type = librenms_device_type - existing_device.save() - logger.info(f"Linked device '{existing_device.name}' to LibreNMS ID {librenms_id}") - - elif action == "update": - # Update hostname, serial, and link to LibreNMS - resolved_name = validation.get("resolved_name") - incoming_serial = libre_device.get("serial") or "" - if resolved_name: - hostname = resolved_name - else: - use_sysname, strip_domain = _resolve_naming_preferences(request) - hostname = _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) - if incoming_serial and incoming_serial != "-": - conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() - if conflict_device: + if conflict_exists: return HttpResponse( - f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " - f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", + f"LibreNMS ID conflict: ID {librenms_id} is already assigned to another device.", status=409, ) - existing_device.serial = incoming_serial - existing_device.name = hostname - if librenms_device_type: - existing_device.device_type = librenms_device_type - set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) - existing_device.save() - logger.info( - f"Updated device '{existing_device.name}': serial={incoming_serial}, " - f"linked to LibreNMS ID {librenms_id}" - ) - elif action == "update_serial": - # Update only the serial and link to LibreNMS - incoming_serial = libre_device.get("serial") or "" - if incoming_serial and incoming_serial != "-": - conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() - if conflict_device: - return HttpResponse( - f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " - f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", - status=409, + if action == "link": + # Link to LibreNMS and update name from LibreNMS data + resolved_name = validation.get("resolved_name") + if resolved_name: + hostname = resolved_name + else: + use_sysname, strip_domain = _resolve_naming_preferences(request) + hostname = _determine_device_name( + libre_device, use_sysname=use_sysname, strip_domain=strip_domain + ) + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) + existing_device.name = hostname + if librenms_device_type: + existing_device.device_type = librenms_device_type + existing_device.save() + logger.info(f"Linked device '{existing_device.name}' to LibreNMS ID {librenms_id}") + + elif action == "update": + # Update hostname, serial, and link to LibreNMS + resolved_name = validation.get("resolved_name") + incoming_serial = libre_device.get("serial") or "" + if resolved_name: + hostname = resolved_name + else: + use_sysname, strip_domain = _resolve_naming_preferences(request) + hostname = _determine_device_name( + libre_device, use_sysname=use_sysname, strip_domain=strip_domain + ) + if incoming_serial and incoming_serial != "-": + conflict_device = ( + Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + ) + if conflict_device: + return HttpResponse( + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", + status=409, + ) + existing_device.serial = incoming_serial + existing_device.name = hostname + if librenms_device_type: + existing_device.device_type = librenms_device_type + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) + existing_device.save() + logger.info( + f"Updated device '{existing_device.name}': serial={incoming_serial}, " + f"linked to LibreNMS ID {librenms_id}" + ) + + elif action == "update_serial": + # Update only the serial and link to LibreNMS + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-": + conflict_device = ( + Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + ) + if conflict_device: + return HttpResponse( + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", + status=409, + ) + existing_device.serial = incoming_serial + if librenms_device_type: + existing_device.device_type = librenms_device_type + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) + existing_device.save() + logger.info( + f"Updated serial on device '{existing_device.name}' to {incoming_serial}, " + f"linked to LibreNMS ID {librenms_id}" ) - existing_device.serial = incoming_serial - if librenms_device_type: - existing_device.device_type = librenms_device_type - set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) - existing_device.save() - logger.info( - f"Updated serial on device '{existing_device.name}' to {incoming_serial}, " - f"linked to LibreNMS ID {librenms_id}" - ) elif action == "sync_name": # Sync device name from LibreNMS (e.g., IP → sysName) From a481d2c2a597b8842dbce7ff0c18533feb84b44e Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 17:11:54 +0100 Subject: [PATCH 18/27] fix: batch 7 - RQ-first cancellation, VC position off-by-one, platform_synced bool - bulk_import_devices_shared: check RQ/Redis job state first (is_stopped/is_failed) before falling back to DB status, matching pattern in process_device_filters(); also check on idx==1 (first iteration) not just idx%5==0 - virtual_chassis._clone_virtual_chassis_data: position fallback now idx+1 (1-based) instead of idx (0-based) so position 0 is never produced - virtual_chassis.detect_virtual_chassis_from_inventory: suggested_name uses position directly (not position+1) since position is already 1-based after batch 6 fallback fix - actions._build_sync_info: wrap platform_synced expression in bool() so lazy and/or chain never returns None when matching_platform is None or netbox_platform is None; all sync flag types are now stable booleans - tests: add TestVCPositionHandling (4 tests) and TestBulkImportCancellation (4 tests); add 2 TestBuildSyncInfo tests for platform_synced=False type-stability --- .../import_utils/bulk_import.py | 44 ++-- .../import_utils/virtual_chassis.py | 11 +- .../tests/test_import_utils.py | 189 ++++++++++++++++++ .../views/imports/actions.py | 2 +- 4 files changed, 226 insertions(+), 20 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index b515a46045..127380a97c 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -94,20 +94,36 @@ def bulk_import_devices_shared( api = LibreNMSAPI(server_key=server_key) for idx, device_id in enumerate(device_ids, start=1): - # Check for job cancellation every 5 devices - if job and idx % 5 == 0: - # Refresh job from DB to get current status - job.job.refresh_from_db() - job_status = job.job.status - status_value = job_status.value if hasattr(job_status, "value") else job_status - if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): - if job.logger: - job.logger.warning(f"Import job cancelled at device {idx} of {total}") - else: - logger.warning(f"Import cancelled at device {idx} of {total}") - break - # Log progress - if job.logger: + # Check for job cancellation on first iteration and every 5th thereafter. + # Check RQ/Redis state first (reflects stop API immediately); fall back to DB. + if job and (idx == 1 or idx % 5 == 0): + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + if rq_job.is_failed or rq_job.is_stopped: + if job.logger: + job.logger.warning( + f"Import job stopped at device {idx} of {total} (RQ status: {rq_job.get_status()})" + ) + else: + logger.warning(f"Import cancelled at device {idx} of {total}") + break + except Exception: + # Fall back to DB check if RQ is unavailable + job.job.refresh_from_db() + job_status = job.job.status + status_value = job_status.value if hasattr(job_status, "value") else job_status + if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): + if job.logger: + job.logger.warning(f"Import job cancelled at device {idx} of {total}") + else: + logger.warning(f"Import cancelled at device {idx} of {total}") + break + # Log progress (only after first check) + if idx > 1 and job.logger: job.logger.info(f"Imported device {idx} of {total}") try: diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py index 2f030875e4..a3c474d393 100644 --- a/netbox_librenms_plugin/import_utils/virtual_chassis.py +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -32,11 +32,11 @@ def _clone_virtual_chassis_data(data: dict | None) -> dict: members = [] for idx, member in enumerate(data.get("members", [])): member_copy = member.copy() - raw_position = member_copy.get("position", idx) + raw_position = member_copy.get("position", idx + 1) try: member_copy["position"] = int(raw_position) except (TypeError, ValueError): - member_copy["position"] = idx + member_copy["position"] = idx + 1 # 1-based fallback; position 0 is invalid members.append(member_copy) member_count = data.get("member_count") or len(members) @@ -214,11 +214,12 @@ def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> d "description": chassis.get("entPhysicalDescr", ""), } - # Generate suggested name if we have master name + # Generate suggested name if we have master name. + # position is already 1-based, so pass it directly (no +1). if master_name: - member_data["suggested_name"] = _generate_vc_member_name(master_name, position + 1) + member_data["suggested_name"] = _generate_vc_member_name(master_name, position) else: - member_data["suggested_name"] = f"Member-{position + 1}" + member_data["suggested_name"] = f"Member-{position}" members.append(member_data) diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 39ea7e72a1..932af5a346 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -2349,6 +2349,50 @@ def test_platform_out_of_sync(self): assert result["platform_synced"] is False assert result["all_synced"] is False + def test_platform_no_match_found_returns_bool(self): + """When find_matching_platform returns no match, platform_synced must be False (not None).""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "ABC123" + existing.platform = MagicMock() # device has a platform set + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + libre_device = {"serial": "ABC123", "os": "ios", "hardware": "-"} + + with patch("netbox_librenms_plugin.utils.find_matching_platform") as mock_platform_match: + mock_platform_match.return_value = {"found": False, "platform": None} + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + # Without bool() cast this would be None; verify it's exactly False (type-stable) + assert result["platform_synced"] is False + assert isinstance(result["platform_synced"], bool) + + def test_platform_synced_no_netbox_platform_returns_bool(self): + """When device has no platform in NetBox and os is non-dash, platform_synced must be bool.""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "ABC123" + existing.platform = None # no platform on device + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + libre_device = {"serial": "ABC123", "os": "eos", "hardware": "-"} + + with patch("netbox_librenms_plugin.utils.find_matching_platform") as mock_platform_match: + mock_platform_match.return_value = {"found": True, "platform": MagicMock()} + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + # None and ... returns None; bool() cast ensures False + assert result["platform_synced"] is False + assert isinstance(result["platform_synced"], bool) + def test_hardware_no_match_device_type_out_of_sync(self): """When hardware is present but no device type match found, device_type_synced is False.""" from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView @@ -2711,3 +2755,148 @@ def test_bulk_import_devices_uses_resolved_server_key(self): # The resolved api.server_key ("resolved-key") must be passed, not None assert mock_import.call_args is not None assert mock_import.call_args.kwargs.get("server_key") == "resolved-key" + + +class TestVCPositionHandling: + """Test VC position normalization and suggested name generation.""" + + def test_clone_vc_data_position_fallback_is_one_based(self): + """_clone_virtual_chassis_data fallback must be 1-based (idx+1, not idx).""" + from netbox_librenms_plugin.import_utils.virtual_chassis import _clone_virtual_chassis_data + + data = {"is_stack": True, "member_count": 2, "members": [{"serial": "S1"}, {"serial": "S2"}]} + result = _clone_virtual_chassis_data(data) + positions = [m["position"] for m in result["members"]] + # First member: idx=0 → position should be 1, not 0 + assert positions[0] == 1 + assert positions[1] == 2 + + def test_clone_vc_data_preserves_explicit_positions(self): + """_clone_virtual_chassis_data must preserve explicitly set positions.""" + from netbox_librenms_plugin.import_utils.virtual_chassis import _clone_virtual_chassis_data + + data = { + "is_stack": True, + "member_count": 2, + "members": [{"serial": "S1", "position": 3}, {"serial": "S2", "position": 5}], + } + result = _clone_virtual_chassis_data(data) + assert result["members"][0]["position"] == 3 + assert result["members"][1]["position"] == 5 + + def test_clone_vc_data_bad_position_falls_back_to_one_based(self): + """_clone_virtual_chassis_data falls back to idx+1 for non-int position.""" + from netbox_librenms_plugin.import_utils.virtual_chassis import _clone_virtual_chassis_data + + data = { + "is_stack": True, + "member_count": 2, + "members": [{"serial": "S1", "position": "bad"}, {"serial": "S2", "position": None}], + } + result = _clone_virtual_chassis_data(data) + # idx=0 → fallback 1, idx=1 → fallback 2 + assert result["members"][0]["position"] == 1 + assert result["members"][1]["position"] == 2 + + def test_suggested_name_uses_position_directly(self): + """Suggested name generation must use position directly (not position+1). + + This test verifies that _generate_vc_member_name is called with the + already-1-based position value, not position+1. + """ + from netbox_librenms_plugin.import_utils.virtual_chassis import _generate_vc_member_name + + # position=1 should produce name with "1", not "2" + name = _generate_vc_member_name("switch-1", 1, pattern="-M{position}") + assert name == "switch-1-M1", f"Expected 'switch-1-M1', got '{name}'" + + # position=2 should produce "2", not "3" + name = _generate_vc_member_name("switch-1", 2, pattern="-M{position}") + assert name == "switch-1-M2", f"Expected 'switch-1-M2', got '{name}'" + + +class TestBulkImportCancellation: + """Test that bulk_import_devices_shared respects RQ and DB cancellation.""" + + def _run_bulk_import(self, mock_rq_job=None, db_status="running", device_ids=None): + """Helper: run bulk_import with provided mocks, return import call count.""" + from unittest.mock import MagicMock, patch + + if device_ids is None: + device_ids = [1, 2, 3, 4, 5, 6] + + job = MagicMock() + job.job.job_id = "test-uuid" + job_status = MagicMock() + job_status.value = db_status + job.job.status = job_status + job.logger = MagicMock() + + with ( + patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") as mock_api_cls, + patch("netbox_librenms_plugin.import_utils.bulk_import.import_single_device") as mock_import, + patch("netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import"), + patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions"), + # Inline imports in the loop use django_rq.get_queue / rq.job.Job directly + patch("django_rq.get_queue") as mock_get_queue, + patch("rq.job.Job") as mock_rqjob_cls, + ): + mock_api = MagicMock() + mock_api.server_key = "default" + mock_api.get_device_info.return_value = (True, {"device_id": 1, "hostname": "sw"}) + mock_api_cls.return_value = mock_api + mock_import.return_value = {"success": True, "device": MagicMock(), "is_vm": False} + + if mock_rq_job is not None: + mock_conn = MagicMock() + mock_queue = MagicMock() + mock_queue.connection = mock_conn + mock_get_queue.return_value = mock_queue + mock_rqjob_cls.fetch.return_value = mock_rq_job + else: + # Simulate RQ unavailable — get_queue raises, triggers DB fallback + mock_get_queue.side_effect = Exception("RQ unavailable") + + from netbox_librenms_plugin.import_utils.bulk_import import bulk_import_devices_shared + + bulk_import_devices_shared(device_ids, user=MagicMock(), server_key=None, job=job) + + return mock_import.call_count + + def test_rq_stopped_cancels_import_loop(self): + """When RQ job is_stopped, import loop should break early.""" + rq_job = MagicMock() + rq_job.is_stopped = True + rq_job.is_failed = False + rq_job.get_status.return_value = "stopped" + + # With 6 devices and RQ stopped on first check (idx=1), at most 1 device processed + count = self._run_bulk_import(mock_rq_job=rq_job, device_ids=[1, 2, 3, 4, 5, 6]) + assert count == 0 # break before first import + + def test_rq_failed_cancels_import_loop(self): + """When RQ job is_failed, import loop should break early.""" + rq_job = MagicMock() + rq_job.is_stopped = False + rq_job.is_failed = True + rq_job.get_status.return_value = "failed" + + count = self._run_bulk_import(mock_rq_job=rq_job, device_ids=[1, 2, 3, 4, 5, 6]) + assert count == 0 + + def test_rq_unavailable_falls_back_to_db_check(self): + """When RQ is unavailable, DB status check is used as fallback.""" + # mock_rq_job=None triggers the side_effect=Exception path + count = self._run_bulk_import(mock_rq_job=None, db_status="failed", device_ids=[1]) + # With DB status "failed", import should not run + assert count == 0 + + def test_healthy_job_runs_all_devices(self): + """When job is healthy, all devices should be imported.""" + rq_job = MagicMock() + rq_job.is_stopped = False + rq_job.is_failed = False + rq_job.get_status.return_value = "started" + + count = self._run_bulk_import(mock_rq_job=rq_job, device_ids=[1, 2, 3]) + assert count == 3 diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index ac724088d3..855081587d 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -788,7 +788,7 @@ def _build_sync_info(libre_device, existing_device): netbox_platform = platform_info["netbox_platform"] matching_platform = platform_info["matching_platform"] - platform_synced = librenms_os == "-" or ( + platform_synced = librenms_os == "-" or bool( netbox_platform and matching_platform and netbox_platform.pk == matching_platform.pk ) From 32caa01980340d765f909f5ace962a87158ba3b5 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Mon, 2 Mar 2026 17:49:59 +0100 Subject: [PATCH 19/27] port: upstream PR#227 maintainer commits to refactor/librenms_id branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Template: cluster/rack badge → text+tick icon (0a24a51 completion) - Template: naming preference badges in Device Information card header (d1c015f) - Template: keep tooltip on device_type match (existing librenms_id feature) - actions.py: _resolve_naming_preferences gains GET support for HTMX hx-include (d1c015f) - actions.py: DeviceValidationDetailsView passes use_sysname/strip_domain to template - actions.py: add _save_device() helper with full_clean()+ValidationError+IntegrityError handling - actions.py: replace all existing_device.save() with _save_device() in conflict actions - device_operations.py: add naming_criteria dict to validate_device_for_import result - device_operations.py: VC-aware name comparison for librenms_id-matched devices - device_operations.py: call update_vc_member_suggested_names when VC is a stack - bulk_import.py: add use_sysname/strip_domain params to process_device_filters - jobs.py: add use_sysname/strip_domain params to FilterDevicesJob.run() - list.py: resolve naming prefs and pass use_sysname/strip_domain to process_device_filters - list.py: pass use_sysname/strip_domain to FilterDevicesJob enqueue - tables/device_status.py: add #use-sysname-toggle, #strip-domain-toggle to all 4 Details hx-include - librenms_import.js: re-init Bootstrap tooltips after modal HTMX content swap (d1c015f) - tests: add TestNameMatchesWithNamingPreferences (8 tests: VC-aware name matching) - tests: fix existing MagicMock devices to set virtual_chassis=None/vc_position=None --- .../import_utils/bulk_import.py | 6 + .../import_utils/device_operations.py | 31 ++- netbox_librenms_plugin/jobs.py | 6 + .../js/librenms_import.js | 6 + .../tables/device_status.py | 8 +- .../htmx/device_validation_details.html | 44 +++- .../tests/test_import_utils.py | 236 ++++++++++++++++++ .../views/imports/actions.py | 55 +++- netbox_librenms_plugin/views/imports/list.py | 40 ++- 9 files changed, 400 insertions(+), 32 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 127380a97c..de819f1f29 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -340,6 +340,8 @@ def process_device_filters( job=None, request=None, return_cache_status: bool = False, + use_sysname: bool = True, + strip_domain: bool = False, ) -> List[dict] | tuple[List[dict], bool]: """ Process LibreNMS device filters and return validated devices. @@ -358,6 +360,8 @@ def process_device_filters( job: Optional JobRunner instance for logging job events request: Optional Django request for client disconnect detection (synchronous only) return_cache_status: When True, returns (devices, from_cache) tuple + use_sysname: If True, prefer sysName over hostname for device name resolution + strip_domain: If True, strip domain suffix from device name Returns: List[dict]: Validated devices with _validation key, or tuple of (devices, from_cache) @@ -511,6 +515,8 @@ def _is_active(d): include_vc_detection=vc_detection_enabled, force_vc_refresh=clear_cache, server_key=api.server_key, + use_sysname=use_sysname, + strip_domain=strip_domain, ) except (BrokenPipeError, ConnectionError, IOError) as e: if request: diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index f7dbb7e1bc..0420b87447 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -16,7 +16,12 @@ match_librenms_hardware_to_device_type, ) from .cache import get_import_device_cache_key -from .virtual_chassis import empty_virtual_chassis_data, get_virtual_chassis_data +from .virtual_chassis import ( + _generate_vc_member_name, + empty_virtual_chassis_data, + get_virtual_chassis_data, + update_vc_member_suggested_names, +) logger = logging.getLogger(__name__) @@ -197,6 +202,7 @@ def validate_device_for_import( "rack": None, "available_racks": [], }, + "naming_criteria": None, # Populated after resolved_name is set } try: @@ -210,6 +216,13 @@ def validate_device_for_import( device_id=librenms_id, ) result["resolved_name"] = hostname + result["naming_criteria"] = { + "use_sysname": use_sysname, + "strip_domain": strip_domain, + "raw_sysname": libre_device.get("sysName") or "", + "raw_hostname": libre_device.get("hostname") or "", + "source": "sysname" if use_sysname and libre_device.get("sysName") else "hostname", + } logger.debug( f"Checking for existing device/VM: " f"librenms_id={librenms_id} (type={type(librenms_id).__name__}), " @@ -271,8 +284,19 @@ def validate_device_for_import( if isinstance(existing_device.custom_field_data.get("librenms_id"), int): result["librenms_id_needs_migration"] = True - # Check if name matches resolved name (accounts for use_sysname/strip_domain) - if hostname and existing_device.name == hostname: + # Check if name matches resolved name (VC-aware: compare against VC member name) + if hostname and existing_device.virtual_chassis and existing_device.vc_position: + vc_expected_name = _generate_vc_member_name( + hostname, + existing_device.vc_position, + serial=existing_device.serial or "", + ) + if existing_device.name == vc_expected_name: + result["name_matches"] = True + else: + result["name_sync_available"] = True + result["suggested_name"] = vc_expected_name + elif hostname and existing_device.name == hostname: result["name_matches"] = True elif hostname and existing_device.name != hostname: result["name_sync_available"] = True @@ -537,6 +561,7 @@ def validate_device_for_import( f"Virtual chassis CONFIRMED for device {hostname}: " f"{vc_detection['member_count']} members" ) + result["virtual_chassis"] = update_vc_member_suggested_names(vc_detection, hostname) except Exception as e: logger.exception(f"Exception during VC detection for device {hostname}: {e}") result["virtual_chassis"]["detection_error"] = str(e) diff --git a/netbox_librenms_plugin/jobs.py b/netbox_librenms_plugin/jobs.py index bb3d00e37d..45422ed4f7 100644 --- a/netbox_librenms_plugin/jobs.py +++ b/netbox_librenms_plugin/jobs.py @@ -46,6 +46,8 @@ def run( show_disabled, exclude_existing=False, server_key=None, + use_sysname=True, + strip_domain=False, **kwargs, ): """ @@ -60,6 +62,8 @@ def run( show_disabled: Whether to include disabled devices exclude_existing: Whether to exclude devices that already exist in NetBox server_key: Optional LibreNMS server key for multi-server setups + use_sysname: If True, prefer sysName over hostname + strip_domain: If True, strip domain suffix from device name **kwargs: Additional job parameters """ from netbox_librenms_plugin.import_utils import process_device_filters @@ -88,6 +92,8 @@ def run( show_disabled=show_disabled, exclude_existing=exclude_existing, job=self, + use_sysname=use_sysname, + strip_domain=strip_domain, ) # Store device IDs for result retrieval diff --git a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js index 728406c703..2e130b1284 100644 --- a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js +++ b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js @@ -1094,6 +1094,12 @@ } showModal(modalElement, fallbackBackdropRef); + + // Re-initialize tooltips for newly swapped modal content + if (typeof bootstrap !== 'undefined' && bootstrap.Tooltip) { + const tooltipEls = modalContent.querySelectorAll('[data-bs-toggle="tooltip"]'); + [...tooltipEls].map(el => new bootstrap.Tooltip(el)); + } } document.body.addEventListener('htmx:afterSwap', ensureModalVisible); diff --git a/netbox_librenms_plugin/tables/device_status.py b/netbox_librenms_plugin/tables/device_status.py index 58eeb7c2ad..3f03764cef 100644 --- a/netbox_librenms_plugin/tables/device_status.py +++ b/netbox_librenms_plugin/tables/device_status.py @@ -495,7 +495,7 @@ def render_actions(self, value, record): f'class="btn btn-sm {btn_class}" ' f"{aria_attr}" f'hx-get="{details_url}" ' - f'hx-include="[name=cluster_{device_id}], [name=role_{device_id}], [name=rack_{device_id}]" ' + f'hx-include="[name=cluster_{device_id}], [name=role_{device_id}], [name=rack_{device_id}], #use-sysname-toggle, #strip-domain-toggle" ' f'hx-target="#htmx-modal-content" ' f'hx-swap="innerHTML" ' f'title="{btn_title}">' @@ -517,7 +517,7 @@ def render_actions(self, value, record): f'