diff --git a/netbox_librenms_plugin/forms.py b/netbox_librenms_plugin/forms.py index f3e3d075e5..d33e94d4ca 100644 --- a/netbox_librenms_plugin/forms.py +++ b/netbox_librenms_plugin/forms.py @@ -581,7 +581,8 @@ def _populate_librenms_locations(self): try: # Use caching to avoid repeated API calls - cache_key = "librenms_locations_choices" + api = LibreNMSAPI() + cache_key = f"librenms_locations_choices:{api.server_key}" cached_choices = cache.get(cache_key) if cached_choices: @@ -589,7 +590,6 @@ def _populate_librenms_locations(self): return # Fetch locations from LibreNMS - api = LibreNMSAPI() success, locations = api.get_locations() if success and locations: diff --git a/netbox_librenms_plugin/import_utils.py b/netbox_librenms_plugin/import_utils.py deleted file mode 100644 index cb9c9a99e7..0000000000 --- a/netbox_librenms_plugin/import_utils.py +++ /dev/null @@ -1,2334 +0,0 @@ -""" -Utilities for importing devices from LibreNMS to NetBox. - -This module provides functions for: -- Validating LibreNMS devices for import -- Retrieving filtered LibreNMS devices -- Importing single and multiple devices -- Smart matching of NetBox objects -- Permission checking for import operations -""" - -import logging -from typing import List - -from core.choices import JobStatusChoices -from dcim.models import Device, DeviceRole, DeviceType, Rack, Site, VirtualChassis -from django.core.cache import cache -from django.core.exceptions import PermissionDenied -from django.db import transaction -from django.utils import timezone -from virtualization.models import Cluster - -from .librenms_api import LibreNMSAPI -from .utils import ( - find_matching_platform, - find_matching_site, - match_librenms_hardware_to_device_type, -) - -logger = logging.getLogger(__name__) - - -# ============================================================================= -# Permission Check Helpers -# ============================================================================= - - -def check_user_permissions(user, permissions): - """ - Check if user has all required permissions. - - Args: - user: The user object to check permissions for - permissions: List of permission strings (e.g., ['dcim.add_device', 'dcim.add_interface']) - - Returns: - tuple: (has_all_permissions: bool, missing_permissions: list[str]) - - Raises: - PermissionDenied: If user is None (no user context available) - """ - if user is None: - raise PermissionDenied("No user context available for permission check") - - missing = [perm for perm in permissions if not user.has_perm(perm)] - return (len(missing) == 0, missing) - - -def require_permissions(user, permissions, action_description="perform this action"): - """ - Require user has all permissions, raising PermissionDenied if not. - - Args: - user: The user object to check permissions for - permissions: List of permission strings - action_description: Human-readable description for error message - - Raises: - PermissionDenied: If user lacks any required permission - """ - has_perms, missing = check_user_permissions(user, permissions) - if not has_perms: - missing_str = ", ".join(missing) - raise PermissionDenied( - f"You do not have permission to {action_description}. Missing permissions: {missing_str}" - ) - - -def get_cache_metadata_key(server_key: str, filters: dict, vc_enabled: bool) -> str: - """ - Generate a consistent cache metadata key from filter parameters. - - Args: - server_key: LibreNMS server identifier - filters: Filter dictionary - vc_enabled: Whether VC detection is enabled - - Returns: - str: Consistent cache key for metadata - """ - # Sort filter items to ensure consistent key generation - filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v) - return f"librenms_filter_cache_metadata_{server_key}_{filter_parts}_{vc_enabled}" - - -def get_active_cached_searches(server_key: str) -> list[dict]: - """ - Retrieve all active cached searches for a server and enrich with display-friendly values. - - Enriches raw filter IDs with human-readable names by looking up location names - from cached choices and converting type codes to display names. - - Args: - server_key: LibreNMS server identifier - - Returns: - List of dicts containing cache metadata with enriched display_filters - """ - from datetime import datetime, timezone - - cache_index_key = f"librenms_cache_index_{server_key}" - cache_index = cache.get(cache_index_key, []) - - active_searches = [] - valid_cache_keys = [] - - # Get location and type choices for enriching display - location_choices = {} - type_choices = { - "": "All Types", - "network": "Network", - "server": "Server", - "storage": "Storage", - "wireless": "Wireless", - "firewall": "Firewall", - "power": "Power", - "appliance": "Appliance", - "printer": "Printer", - "loadbalancer": "Load Balancer", - "other": "Other", - } - - # Get cached location choices for enrichment - location_cache_key = "librenms_locations_choices" - cached_locations = cache.get(location_cache_key) - if cached_locations: - location_choices = dict(cached_locations) - - for cache_key in cache_index: - metadata = cache.get(cache_key) - if metadata: - # Cache still exists, calculate time remaining - cached_at = datetime.fromisoformat(metadata.get("cached_at")) - cache_timeout = metadata.get("cache_timeout", 300) - now = datetime.now(timezone.utc) - age_seconds = (now - cached_at).total_seconds() - remaining_seconds = max(0, cache_timeout - age_seconds) - - if remaining_seconds > 0: - # Add remaining time and cache key - metadata["remaining_seconds"] = int(remaining_seconds) - metadata["cache_key"] = cache_key - - # Enrich filters with human-readable display values - if "filters" in metadata: - display_filters = metadata["filters"].copy() - # Convert location ID to location name - if "location" in display_filters and display_filters["location"] in location_choices: - display_filters["location"] = location_choices[display_filters["location"]] - # Convert type code to display name - if "type" in display_filters and display_filters["type"] in type_choices: - display_filters["type"] = type_choices[display_filters["type"]] - metadata["display_filters"] = display_filters - else: - # Fallback if filters key missing - metadata["display_filters"] = {} - - active_searches.append(metadata) - valid_cache_keys.append(cache_key) - - # Clean up index if any keys have expired - if len(valid_cache_keys) < len(cache_index): - cache.set(cache_index_key, valid_cache_keys, timeout=3600) - - # Sort by most recent first - active_searches.sort(key=lambda x: x.get("cached_at", ""), reverse=True) - - return active_searches - - -def get_validated_device_cache_key(server_key: str, filters: dict, device_id: int | str, vc_enabled: bool) -> str: - """ - Generate a consistent cache key for validated device data. - - This ensures both synchronous and background job processing use the same - cache keys, avoiding duplicate validation work and cache entries. - - Args: - server_key: LibreNMS server key - filters: Filter dict with location, type, os, hostname, sysname, hardware keys - device_id: LibreNMS device ID - vc_enabled: Whether virtual chassis detection was enabled - - Returns: - str: Cache key for the validated device - - Example: - >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) - >>> key - 'validated_device_default_-1234567890_123_vc' - """ - # Sort filters for consistent hashing - filter_hash = hash(str(sorted(filters.items()))) - vc_part = "vc" if vc_enabled else "novc" - return f"validated_device_{server_key}_{filter_hash}_{device_id}_{vc_part}" - - -def get_import_device_cache_key(device_id: int | str, server_key: str = "default") -> str: - """ - Generate cache key for raw LibreNMS device data. - - This key is used to cache raw device data (without validation metadata) - to avoid redundant API calls when users interact with dropdowns during - the import workflow. - - Args: - device_id: LibreNMS device ID - server_key: LibreNMS server identifier for multi-server setups - - Returns: - str: Cache key for the device data - - Example: - >>> get_import_device_cache_key(123, "production") - 'import_device_data_production_123' - """ - return f"import_device_data_{server_key}_{device_id}" - - -def _determine_device_name( - libre_device: dict, - use_sysname: bool = True, - strip_domain: bool = False, - device_id: int | str = None, -) -> str: - """ - Determine the device/VM name from LibreNMS data. - - Centralized logic for building device names with consistent handling of: - - sysName vs hostname preference - - Domain stripping (avoiding IP addresses) - - Fallback to device_id when name is missing - - Args: - libre_device: Device data from LibreNMS - use_sysname: If True, prefer sysName; if False, use hostname - strip_domain: If True, strip domain suffix (e.g., '.example.com') - device_id: LibreNMS device ID for fallback name generation - - Returns: - str: The determined device name - - Example: - >>> _determine_device_name({'sysName': 'router.example.com', 'hostname': 'router'}, - ... use_sysname=True, strip_domain=True) - 'router' - """ - # Determine base name based on use_sysname preference - if use_sysname: - name = libre_device.get("sysName") or libre_device.get("hostname") - else: - name = libre_device.get("hostname") or libre_device.get("sysName") - - # Fallback to device_id if no name found - if not name: - if device_id is not None: - name = f"device-{device_id}" - else: - name = libre_device.get("device_id", "unknown") - name = f"device-{name}" - - # Strip domain if requested (but not for IP addresses) - if strip_domain and name and "." in name: - try: - from ipaddress import ip_address - - ip_address(name) - # It's a valid IP address, don't strip - except ValueError: - # Not an IP, safe to strip domain - name = name.split(".")[0] - - return name - - -def empty_virtual_chassis_data() -> dict: - """Public helper for callers that need a blank VC payload.""" - - return { - "is_stack": False, - "member_count": 0, - "members": [], - "detection_error": None, - } - - -def _clone_virtual_chassis_data(data: dict | None) -> dict: - """Return a defensive copy of cached VC data to avoid shared references.""" - - if not data: - return empty_virtual_chassis_data() - - members = [] - for idx, member in enumerate(data.get("members", [])): - member_copy = member.copy() - raw_position = member_copy.get("position", idx) - try: - member_copy["position"] = int(raw_position) - except (TypeError, ValueError): - member_copy["position"] = idx - members.append(member_copy) - - member_count = data.get("member_count") or len(members) - - return { - "is_stack": bool(data.get("is_stack")), - "member_count": member_count, - "members": members, - "detection_error": data.get("detection_error"), - } - - -_VC_CACHE_VERSION = "v1" - - -def _vc_cache_key(api: LibreNMSAPI, device_id: int | str) -> str: - server_key = getattr(api, "server_key", "default") - return f"librenms_vc_detection_{_VC_CACHE_VERSION}_{server_key}_{device_id}" - - -def get_virtual_chassis_data(api: LibreNMSAPI, device_id: int | str, *, force_refresh: bool = False) -> dict: - """Fetch (and cache) virtual chassis data for a LibreNMS device.""" - - if not api or device_id is None: - return empty_virtual_chassis_data() - - cache_key = _vc_cache_key(api, device_id) - if not force_refresh: - cached = cache.get(cache_key) - if cached is not None: - return _clone_virtual_chassis_data(cached) - - detection_data = detect_virtual_chassis_from_inventory(api, device_id) - if detection_data and "detection_error" not in detection_data: - detection_data["detection_error"] = None - - cache_value = _clone_virtual_chassis_data(detection_data) if detection_data else empty_virtual_chassis_data() - - cache_timeout = getattr(api, "cache_timeout", 300) or 300 - cache.set(cache_key, cache_value, timeout=cache_timeout) - return _clone_virtual_chassis_data(cache_value) - - -def prefetch_vc_data_for_devices(api: LibreNMSAPI, device_ids: List[int], *, force_refresh: bool = False) -> None: - """ - Pre-warm the virtual chassis cache for multiple devices. - - This eliminates the 0.5-1s delay when rendering the import table - by proactively fetching VC data before validation. - - Args: - api: LibreNMSAPI instance - device_ids: List of LibreNMS device IDs to prefetch VC data for - force_refresh: When True, bypass cache and fetch fresh data - - Example: - >>> # Before rendering import table - >>> prefetch_vc_data_for_devices(api, [123, 124, 125]) - >>> # Now all validate_device_for_import() calls hit cache instantly - """ - if not api or not device_ids: - return - - logger.debug(f"Pre-warming VC cache for {len(device_ids)} devices") - - for idx, device_id in enumerate(device_ids): - # This populates the cache if empty, or skips if already cached - try: - get_virtual_chassis_data(api, device_id, force_refresh=force_refresh) - except (BrokenPipeError, ConnectionError, IOError, OSError) as e: - logger.warning(f"Connection error during VC prefetch at device {idx}: {e}") - # Stop processing if connection is broken - return - except Exception as e: - # Log but continue for other errors - logger.warning(f"Error prefetching VC data for device {device_id}: {e}") - - logger.debug(f"VC cache warming complete for {len(device_ids)} devices") - - -def get_device_count_for_filters( - api: LibreNMSAPI, - filters: dict, - clear_cache: bool = False, - show_disabled: bool = True, -) -> int: - """ - Get count of LibreNMS devices matching filters. - - This is a lightweight function to determine device count for background job - decision making. Uses the same caching as get_librenms_devices_for_import(). - - Args: - api: LibreNMS API client instance - filters: Filter dict with location, type, os, hostname, sysname keys - clear_cache: Whether to force cache refresh - show_disabled: Whether to include disabled devices - - Returns: - int: Count of devices matching filters - """ - devices = get_librenms_devices_for_import(api, filters=filters, force_refresh=clear_cache) - - # Filter out disabled devices if requested - if not show_disabled: - devices = [d for d in devices if d.get("status") == 1] - - return len(devices) - - -def get_librenms_devices_for_import( - api: LibreNMSAPI = None, - filters: dict = None, - server_key: str = None, - *, - force_refresh: bool = False, - return_cache_status: bool = False, -) -> List[dict] | tuple[List[dict], bool]: - """ - Retrieve LibreNMS devices based on filters. - - Args: - api: LibreNMSAPI instance (if not provided, creates one with server_key) - filters: Dict containing filter parameters: - - location: LibreNMS location/site filter - - type: Device type filter - - os: Operating system filter - - hostname: Hostname filter (partial match) - - sysname: System name filter (partial match) - - status: Device status filter (1=up, 0=down) - - disabled: Include disabled devices (0=active only, 1=all) - server_key: Key for specific server configuration (used if api not provided) - force_refresh: When True, bypass the cache and fetch fresh data - return_cache_status: When True, returns (devices, from_cache) tuple - - Returns: - List of device dictionaries from LibreNMS, or tuple of (devices, from_cache) - if return_cache_status is True. from_cache=True means data was loaded from - existing cache; from_cache=False means data was just fetched from LibreNMS. - """ - try: - # Use provided API instance or create a new one - if api is None: - api = LibreNMSAPI(server_key=server_key) - - # Build LibreNMS API filters using the type/query format - # LibreNMS API v0 expects ?type=X&query=Y format, not direct parameters - # NOTE: API only supports ONE type/query pair, so we'll use the most - # specific filter for the API and apply others client-side - api_filters = {} - client_filters = {} # Filters to apply after fetching from API - - if filters: - # Check for status filter first - it has special handling - if filters.get("status") is not None: - # Status filter uses special types that don't need query param - if filters["status"] == 1: - api_filters["type"] = "up" - elif filters["status"] == 0: - api_filters["type"] = "down" - - # Save ALL other filters for client-side filtering when status is used - if filters.get("location"): - client_filters["location"] = filters["location"] - if filters.get("type"): - client_filters["type"] = filters["type"] - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - else: - # Priority order for type/query filters: location > type > os > hostname > sysname - # Note: When sysname is combined with other filters, it's applied client-side for partial matching - # When sysname is alone, it uses API exact match (type=sysName) - # Note: hardware is always applied client-side for partial matching - # Use first available for API, save others for client-side filtering - if filters.get("location"): - api_filters["type"] = "location_id" - api_filters["query"] = filters["location"] - # Save remaining filters for client-side - if filters.get("type"): - client_filters["type"] = filters["type"] - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("type"): - api_filters["type"] = "type" - api_filters["query"] = filters["type"] - # Save remaining filters for client-side - if filters.get("os"): - client_filters["os"] = filters["os"] - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("os"): - api_filters["type"] = "os" - api_filters["query"] = filters["os"] - # Save remaining filters for client-side - if filters.get("hostname"): - client_filters["hostname"] = filters["hostname"] - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("hostname"): - api_filters["type"] = "hostname" - api_filters["query"] = filters["hostname"] - # Save sysname and hardware for client-side - if filters.get("sysname"): - client_filters["sysname"] = filters["sysname"] - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("sysname"): - # sysname-only filter: Use API exact match (type=sysName&query=) - # This is safe - returns empty if no exact match found - api_filters["type"] = "sysName" - api_filters["query"] = filters["sysname"] - # Save hardware for client-side - if filters.get("hardware"): - client_filters["hardware"] = filters["hardware"] - elif filters.get("hardware"): - # hardware-only filter: apply client-side for partial matching - client_filters["hardware"] = filters["hardware"] - - # Note: disabled filter isn't directly supported by LibreNMS API - # We'll filter client-side if needed - - # Use caching to avoid repeated API calls - # Include both API and client filters in cache key - cache_key = f"librenms_devices_import_{server_key}_{hash(str(api_filters))}_{hash(str(client_filters))}" - from_cache = False - - if force_refresh: - cache.delete(cache_key) - else: - cached_result = cache.get(cache_key) - if cached_result is not None: - # No need to deepcopy - cached data isn't mutated - devices = cached_result - from_cache = True - if return_cache_status: - return devices, from_cache - return devices - - success, devices = api.list_devices(api_filters if api_filters else None) - - if not success: - logger.error(f"Failed to retrieve devices from LibreNMS: {devices}") - if return_cache_status: - return [], False - return [] - - # Apply client-side filters if any - if client_filters: - devices = _apply_client_filters(devices, client_filters) - - # Cache using configured timeout (default 300s) - # No need to deepcopy - Django's cache backend handles serialization - cache.set(cache_key, devices, timeout=api.cache_timeout) - - if return_cache_status: - return devices, from_cache - return devices - - except Exception: - logger.exception("Error retrieving LibreNMS devices for import") - if return_cache_status: - return [], False - return [] - - -def _apply_client_filters(devices: List[dict], filters: dict) -> List[dict]: - """ - Apply client-side filters to device list. - - Args: - devices: List of device dicts from LibreNMS - filters: Dict of filters to apply (location, type, os, hostname, sysname) - - Returns: - Filtered list of devices - """ - filtered = devices - - if filters.get("location"): - location_id = str(filters["location"]) - filtered = [d for d in filtered if str(d.get("location_id", "")) == location_id] - - if filters.get("type"): - device_type = filters["type"].lower() - filtered = [d for d in filtered if d.get("type", "").lower() == device_type] - - if filters.get("os"): - os_filter = filters["os"].lower() - filtered = [d for d in filtered if os_filter in d.get("os", "").lower()] - - if filters.get("hostname"): - hostname_filter = filters["hostname"].lower() - filtered = [d for d in filtered if hostname_filter in d.get("hostname", "").lower()] - - if filters.get("sysname"): - sysname_filter = filters["sysname"].lower() - filtered = [d for d in filtered if sysname_filter in d.get("sysName", "").lower()] - - if filters.get("hardware"): - hardware_filter = filters["hardware"].lower() - filtered = [d for d in filtered if hardware_filter in (d.get("hardware") or "").lower()] - - return filtered - - -def validate_device_for_import( - libre_device: dict, - import_as_vm: bool = False, - api: "LibreNMSAPI" = None, - *, - include_vc_detection: bool = True, - force_vc_refresh: bool = False, -) -> dict: - """ - Validate if a LibreNMS device can be imported to NetBox. - - Performs comprehensive validation: - - Checks if device already exists in NetBox - - Validates required prerequisites (Site, DeviceType, DeviceRole for devices) - OR (Cluster for VMs) - - Provides smart matching for missing objects - - Detects virtual chassis/stack configuration (if API provided) - - Returns detailed validation status - - Args: - libre_device: Device data from LibreNMS - import_as_vm: If True, validate for VM import instead of device import - api: Optional LibreNMSAPI instance for virtual chassis detection - include_vc_detection: Skip VC detection when False to speed up bulk operations - force_vc_refresh: When True, bypass cached VC data and re-query LibreNMS - - Returns: - dict: Validation result with structure: - { - 'is_ready': bool, # Can import without user intervention - 'can_import': bool, # Can import (possibly after configuration) - 'import_as_vm': bool, # Whether importing as VM - 'existing_device': Device or VirtualMachine or None, - 'issues': List[str], # Blocking issues - 'warnings': List[str], # Non-blocking warnings - 'site': { # Only for devices - 'found': bool, - 'site': Site or None, - 'match_type': str, # 'exact' or None - 'suggestions': List[Site] # Alternative suggestions - }, - 'device_type': { # Only for devices - 'found': bool, - 'device_type': DeviceType or None, - 'match_type': str, # 'exact' or None - 'suggestions': List[dict] # Device types for user selection - }, - 'device_role': { # Only for devices - 'found': bool, # Always False - requires manual selection - 'role': DeviceRole or None, - 'available_roles': List[DeviceRole] # All roles for user selection - }, - 'cluster': { # Only for VMs - 'found': bool, # Always False - requires manual selection - 'cluster': Cluster or None, - 'available_clusters': List[Cluster] # All clusters for user selection - }, - 'platform': { - 'found': bool, - 'platform': Platform or None, - 'match_type': str # 'exact' or None - } - } - - Example: - >>> validation = validate_device_for_import(libre_device) - >>> if validation['is_ready']: - ... import_single_device(libre_device['device_id']) - """ - result = { - "is_ready": False, - "can_import": False, - "import_as_vm": import_as_vm, - "existing_device": None, - "existing_match_type": None, # Track how existing device was matched - "issues": [], - "warnings": [], - "virtual_chassis": empty_virtual_chassis_data(), - "site": { - "found": False, - "site": None, - "match_type": None, - "suggestions": [], - }, - "device_type": { - "found": False, - "device_type": None, - "match_type": None, - "suggestions": [], - }, - "device_role": { - "found": False, - "role": None, - "available_roles": [], - }, - "cluster": { - "found": False, - "cluster": None, - "available_clusters": [], - }, - "platform": {"found": False, "platform": None, "match_type": None}, - "rack": { - "found": False, - "rack": None, - "available_racks": [], - }, - } - - try: - # 1. Check if device/VM already exists in NetBox - # Always check both Devices AND VMs to properly detect existing objects - librenms_id = libre_device.get("device_id") - hostname = libre_device.get("hostname", "") - logger.debug( - f"Checking for existing device/VM: " - f"librenms_id={librenms_id} (type={type(librenms_id).__name__}), " - f"hostname={hostname}" - ) - - from virtualization.models import VirtualMachine - - # Check for existing VM first (by librenms_id custom field) - # Always query with int to match custom field type - try: - existing_vm = VirtualMachine.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() - except (ValueError, TypeError): - # librenms_id is not convertible to int; no match will be found - existing_vm = None - - if existing_vm: - logger.info(f"Found existing VM: {existing_vm.name} (matched by librenms_id={librenms_id})") - result["existing_device"] = existing_vm - result["existing_match_type"] = "librenms_id" - result["import_as_vm"] = True # Force VM mode since VM exists - result["warnings"].append(f"VM already imported to NetBox as '{existing_vm.name}'") - result["can_import"] = False - return result - - # Check for existing Device (by librenms_id custom field) - # Always query with int to match custom field type - try: - existing_device = Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() - except (ValueError, TypeError): - # librenms_id is not convertible to int; no match will be found - existing_device = None - - if existing_device: - logger.info(f"Found existing device: {existing_device.name} (matched by librenms_id={librenms_id})") - result["existing_device"] = existing_device - result["existing_match_type"] = "librenms_id" - result["warnings"].append(f"Device already imported to NetBox as '{existing_device.name}'") - result["can_import"] = False - return result - - # Check by hostname/name - Check both VMs and Devices for conflicts - existing_vm = VirtualMachine.objects.filter(name__iexact=hostname).first() - existing_device = Device.objects.filter(name__iexact=hostname).first() - - # If BOTH exist with same hostname, it's ambiguous - don't match either - if existing_vm and existing_device: - logger.warning( - f"Hostname conflict: Both VM '{existing_vm.name}' and Device " - f"'{existing_device.name}' exist with hostname '{hostname}'" - ) - result["warnings"].append( - f"Both a VM and Device exist with hostname '{hostname}' in NetBox. " - f"Cannot determine which to match. Please set the librenms_id custom field on the correct object." - ) - # Don't set existing_device, don't block import - let user proceed as new - # This allows them to import and then resolve the conflict manually - elif existing_vm: - logger.info(f"Found existing VM by hostname: {existing_vm.name}") - result["existing_device"] = existing_vm - result["existing_match_type"] = "hostname" - result["import_as_vm"] = True # Force VM mode since VM exists - result["warnings"].append( - f"VM with same hostname exists in NetBox as '{existing_vm.name}' (not linked to LibreNMS)" - ) - result["can_import"] = False - return result - elif existing_device: - logger.info(f"Found existing device by hostname: {existing_device.name}") - result["existing_device"] = existing_device - result["existing_match_type"] = "hostname" - result["warnings"].append( - f"Device with same hostname exists in NetBox as '{existing_device.name}' (not linked to LibreNMS)" - ) - result["can_import"] = False - return result - - # Check by primary IP (weaker match, IP could be reassigned) - only for devices - primary_ip = libre_device.get("ip") - if primary_ip and not import_as_vm: - from ipam.models import IPAddress - - existing_ip = IPAddress.objects.filter(address__startswith=primary_ip).first() - if existing_ip and existing_ip.assigned_object: - device = existing_ip.assigned_object.device if hasattr(existing_ip.assigned_object, "device") else None - if device: - result["existing_device"] = device - result["existing_match_type"] = "primary_ip" - result["warnings"].append( - f"IP address {primary_ip} already assigned to device '{device.name}' (not linked to LibreNMS)" - ) - result["can_import"] = False - return result - - # Validate based on import type (Device or VM) - if import_as_vm: - # 2. For VMs: Validate Cluster (required) - Must be manually selected - from virtualization.models import Cluster - - result["cluster"]["found"] = False - result["issues"].append("Cluster must be manually selected before importing as VM") - # Provide list of available clusters for user selection (cached) - cache_key = "librenms_import_all_clusters" - all_clusters = cache.get(cache_key) - if all_clusters is None: - all_clusters = list(Cluster.objects.all()) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, all_clusters, cache_timeout) - result["cluster"]["available_clusters"] = all_clusters - - # Skip device-specific validations for VMs - result["site"]["found"] = True # Not required for VMs - result["device_type"]["found"] = True # Not required for VMs - result["device_role"]["found"] = True # Not required for VMs - - else: - # 2. For Devices: Validate Site (required) - location = libre_device.get("location", "") - site_match = find_matching_site(location) - result["site"] = site_match - - if not site_match["found"]: - result["issues"].append(f"No matching site found for location: '{location}'") - # Get alternative suggestions - if location: - all_sites = Site.objects.all()[:10] # Limit for performance - result["site"]["suggestions"] = list(all_sites) - - # 3. Validate DeviceType (required) - hardware = libre_device.get("hardware", "") - dt_match = match_librenms_hardware_to_device_type(hardware) - result["device_type"] = dt_match - - if not dt_match["matched"]: - result["issues"].append(f"No matching device type found for hardware: '{hardware}'") - # Get some device types for user to choose from - all_device_types = DeviceType.objects.all()[:10] - result["device_type"]["suggestions"] = [ - { - "device_type": dt, - "similarity": 0.0, # No fuzzy matching, just showing options - "match_field": None, - } - for dt in all_device_types - ] - else: - # Rename 'matched' to 'found' for consistency - result["device_type"]["found"] = dt_match["matched"] - result["device_type"]["device_type"] = dt_match["device_type"] - result["device_type"]["match_type"] = dt_match["match_type"] - - # 4. DeviceRole (required) - Must be manually selected by user - logger.debug(f"[{hostname}] Issues BEFORE adding role issue: {result['issues']}") - result["device_role"]["found"] = False - result["issues"].append("Device role must be manually selected before import") - logger.debug(f"[{hostname}] Issues AFTER adding role issue: {result['issues']}") - # Provide list of available roles for user selection (cached) - cache_key = "librenms_import_all_roles" - all_roles = cache.get(cache_key) - if all_roles is None: - all_roles = list(DeviceRole.objects.all()) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, all_roles, cache_timeout) - result["device_role"]["available_roles"] = all_roles - - # 4b. Rack (optional) - Provide available racks for the matched site - if site_match["found"] and site_match["site"]: - site = site_match["site"] - # Use cache to optimize rack lookups per site - cache_key = f"librenms_import_racks_site_{site.pk}" - available_racks = cache.get(cache_key) - - if available_racks is None: - from dcim.models import Rack - from django.db.models import Q - - # Query racks for this site - include both: - # 1. Racks assigned to locations within the site - # 2. Racks directly assigned to the site (without location) - available_racks = list( - Rack.objects.filter(Q(location__site=site) | Q(site=site)) - .select_related("location", "site") - .order_by("location__name", "name") - ) - # Use API cache timeout if available, otherwise use default 5 minutes - cache_timeout = api.cache_timeout if api else 300 - cache.set(cache_key, available_racks, cache_timeout) - - result["rack"]["available_racks"] = available_racks - # Rack is optional, don't add to issues - result["rack"]["found"] = True # Mark as "found" even if None (optional field) - - # Skip VM-specific validations for devices - result["cluster"]["found"] = True # Not required for devices - - # 5. Match Platform (optional - same for both devices and VMs) - os = libre_device.get("os", "") - platform_match = find_matching_platform(os) - result["platform"] = platform_match - - if not platform_match["found"] and os: - result["warnings"].append(f"No matching platform found for OS: '{os}'") - - # 6. Additional validations - if not hostname: - result["issues"].append("Device has no hostname") - - # Serial number check - serial = libre_device.get("serial", "") - if serial and serial != "-": - existing_serial = Device.objects.filter(serial=serial).first() - if existing_serial: - result["warnings"].append(f"Serial number {serial} already exists on device: {existing_serial.name}") - - # 7. Virtual chassis detection (only for devices, not VMs) - if include_vc_detection and not import_as_vm and api is not None: - device_id = libre_device.get("device_id") - if device_id: - try: - logger.debug(f"Calling get_virtual_chassis_data for device {device_id}") - vc_detection = get_virtual_chassis_data(api, device_id, force_refresh=force_vc_refresh) - logger.debug( - f"VC detection result: is_stack={vc_detection.get('is_stack')}, " - f"member_count={vc_detection.get('member_count')}, " - f"members={len(vc_detection.get('members', []))}" - ) - if vc_detection: - result["virtual_chassis"] = vc_detection - if vc_detection["is_stack"]: - logger.debug( - f"Virtual chassis CONFIRMED for device {hostname}: " - f"{vc_detection['member_count']} members" - ) - except Exception as e: - logger.exception(f"Exception during VC detection for device {hostname}: {e}") - result["virtual_chassis"]["detection_error"] = str(e) - else: - logger.debug(f"No device_id found for {hostname}") - - # 8. Determine if device/VM is ready to import - result["can_import"] = len(result["issues"]) == 0 - - if import_as_vm: - # For VMs: only cluster is required - result["is_ready"] = result["can_import"] and result["cluster"]["found"] - else: - # For Devices: site, device_type, and device_role are required - result["is_ready"] = ( - result["can_import"] - and result["site"]["found"] - and result["device_type"]["found"] - and result["device_role"]["found"] - ) - - logger.debug( - f"Validation for {libre_device.get('hostname')} ({'VM' if import_as_vm else 'Device'}): " - f"issues={len(result['issues'])}, can_import={result['can_import']}, " - f"issues_list={result['issues']}" - ) - - return result - - except Exception as e: - logger.exception(f"Error validating device for import: {libre_device.get('hostname', 'unknown')}") - result["issues"].append(f"Validation error: {str(e)}") - return result - - -def import_single_device( - device_id: int, - server_key: str = None, - validation: dict = None, - manual_mappings: dict = None, - sync_options: dict = None, - libre_device: dict = None, -) -> dict: - """ - Import a single LibreNMS device to NetBox. - - Args: - device_id: LibreNMS device ID - server_key: LibreNMS server configuration key - validation: Pre-computed validation dict (optional) - manual_mappings: Manual object mappings (optional): - - site_id: NetBox Site ID - - device_type_id: NetBox DeviceType ID - - device_role_id: NetBox DeviceRole ID - - platform_id: NetBox Platform ID (optional) - - rack_id: NetBox Rack ID (optional) - sync_options: Sync options (optional): - - sync_interfaces: bool (default True) - - sync_cables: bool (default True) - - sync_ips: bool (default True) - - sync_fields: bool (default True) - libre_device: Pre-fetched LibreNMS device data (optional). - If provided, skips API call to fetch device info. - - Returns: - dict: Import result with structure: - { - 'success': bool, - 'device': Device object or None, - 'message': str, - 'error': str or None, - 'synced': { - 'interfaces': int, - 'cables': int, - 'ip_addresses': int - } - } - """ - try: - api = LibreNMSAPI(server_key=server_key) - - # Use pre-fetched device data if provided, otherwise fetch from API - if libre_device is None: - success, libre_device = api.get_device_info(device_id) - if not success or not libre_device: - return { - "success": False, - "device": None, - "message": "", - "error": f"Failed to retrieve device {device_id} from LibreNMS", - "synced": {}, - } - - # Validate device if validation not provided - if validation is None: - validation = validate_device_for_import(libre_device) - - # Check if device already exists - if validation.get("existing_device"): - return { - "success": False, - "device": validation["existing_device"], - "message": "", - "error": f"Device already exists: {validation['existing_device'].name}", - "synced": {}, - } - - # Use validation-derived matches, allow manual mappings to override specific fields - site = validation["site"].get("site") - device_type = validation["device_type"].get("device_type") - device_role = validation["device_role"].get("role") - platform = validation["platform"].get("platform") - rack = validation.get("rack", {}).get("rack") - - if manual_mappings: - site = Site.objects.filter(id=manual_mappings.get("site_id")).first() or site - device_type = DeviceType.objects.filter(id=manual_mappings.get("device_type_id")).first() or device_type - device_role = DeviceRole.objects.filter(id=manual_mappings.get("device_role_id")).first() or device_role - - platform_id = manual_mappings.get("platform_id") - if platform_id: - from dcim.models import Platform - - platform = Platform.objects.filter(id=platform_id).first() or platform - - rack_id = manual_mappings.get("rack_id") - if rack_id: - rack = Rack.objects.select_related("location", "site").filter(id=rack_id).first() or rack - - rack = rack or validation.get("rack", {}).get("rack") - - # Validate required fields - if not site: - return { - "success": False, - "device": None, - "message": "", - "error": "Site is required but not provided", - "synced": {}, - } - if not device_type: - return { - "success": False, - "device": None, - "message": "", - "error": "Device type is required but not provided", - "synced": {}, - } - if not device_role: - return { - "success": False, - "device": None, - "message": "", - "error": "Device role is required but not provided", - "synced": {}, - } - - # Create device in NetBox - with transaction.atomic(): - # Determine device name based on sync options - use_sysname = sync_options.get("use_sysname", True) if sync_options else True - strip_domain = sync_options.get("strip_domain", False) if sync_options else False - - device_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=strip_domain, - device_id=device_id, - ) - - # Generate import timestamp comment - import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") - - device_data = { - "name": device_name, - "site": site, - "device_type": device_type, - "role": device_role, - "status": "active" if libre_device.get("status") == 1 else "offline", - "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - "custom_field_data": {"librenms_id": int(device_id)}, - } - - # Add optional fields - if platform: - device_data["platform"] = platform - - if rack: - device_data["rack"] = rack - - serial = libre_device.get("serial", "") - if serial and serial != "-": - device_data["serial"] = serial - - location_name = libre_device.get("location", "") - if location_name and location_name != "-": - from dcim.models import Location - - # Try to find matching location within the site - location = Location.objects.filter(site=site, name__iexact=location_name).first() - if location: - device_data["location"] = location - - # Create the device - device = Device(**device_data) - device.full_clean() - device.save() - - # Sync additional data based on options - sync_options = sync_options or {} - synced = {"interfaces": 0, "cables": 0, "ip_addresses": 0} - - try: - # Sync interfaces - if sync_options.get("sync_interfaces", True): - # This is simplified - would need proper request context - # For now, just log that it should be done - logger.info(f"Interface sync should be performed for device {device.name}") - - # Sync cables - if sync_options.get("sync_cables", True): - logger.info(f"Cable sync should be performed for device {device.name}") - - # Sync IP addresses - if sync_options.get("sync_ips", True): - logger.info(f"IP address sync should be performed for device {device.name}") - - except Exception as e: - logger.warning(f"Error during post-import sync: {str(e)}") - # Don't fail the import if sync fails - - return { - "success": True, - "device": device, - "message": f"Successfully imported device: {device.name}", - "error": None, - "synced": synced, - } - - except Exception as e: - logger.exception(f"Error importing device {device_id}") - return { - "success": False, - "device": None, - "message": "", - "error": str(e), - "synced": {}, - } - - -def bulk_import_devices_shared( - device_ids: List[int], - server_key: str = None, - sync_options: dict = None, - manual_mappings_per_device: dict = None, - libre_devices_cache: dict = None, - job=None, - user=None, -) -> dict: - """ - Shared function for importing multiple LibreNMS devices to NetBox. - - Used by both synchronous imports and background jobs. Handles per-device error - collection and optional progress logging when job context is provided. - - Args: - device_ids: List of LibreNMS device IDs to import - server_key: LibreNMS server configuration key - sync_options: Sync options to apply to all devices - manual_mappings_per_device: Dict mapping device_id to manual_mappings dict - Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} - libre_devices_cache: Optional dict mapping device_id to pre-fetched device data - to avoid redundant API calls. Example: {123: {...device_data...}} - job: Optional JobRunner instance for progress logging and cancellation checks - user: User performing the import (for permission checks). If job is provided, - user is extracted from job.job.user if not explicitly passed. - - Returns: - dict: Bulk import result with structure: - { - 'total': int, - 'success': List[dict], # Successfully imported devices - 'failed': List[dict], # Failed imports with errors - 'skipped': List[dict], # Skipped devices (already exist, etc.) - 'virtual_chassis_created': int # Number of VCs created - } - - Raises: - PermissionDenied: If user lacks required permissions - - Example: - >>> # Synchronous usage - >>> result = bulk_import_devices_shared([1, 2, 3, 4, 5], user=request.user) - >>> # Background job usage - >>> result = bulk_import_devices_shared([1, 2, 3], job=self) - """ - # Extract user from job if not explicitly provided - if user is None and job is not None: - user = getattr(job.job, "user", None) - - # Check permissions at start of bulk operation - required_perms = [ - "dcim.add_device", - "dcim.add_interface", - "dcim.add_virtualchassis", - ] - require_permissions(user, required_perms, "import devices") - - total = len(device_ids) - success_list = [] - failed_list = [] - skipped_list = [] - vc_created_count = 0 - processed_vc_domains = set() # Track VCs already created by domain - - # Initialize API client once for all devices to avoid repeated config parsing - api = LibreNMSAPI(server_key=server_key) - - for idx, device_id in enumerate(device_ids, start=1): - # Check for job cancellation every 5 devices - if job and idx % 5 == 0: - # Refresh job from DB to get current status - job.job.refresh_from_db() - job_status = job.job.status - status_value = job_status.value if hasattr(job_status, "value") else job_status - if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): - if job.logger: - job.logger.warning(f"Import job cancelled at device {idx} of {total}") - else: - logger.warning(f"Import cancelled at device {idx} of {total}") - break - # Log progress - if job.logger: - job.logger.info(f"Imported device {idx} of {total}") - - try: - # Use cached device data if available to avoid redundant API calls - if libre_devices_cache and device_id in libre_devices_cache: - libre_device = libre_devices_cache[device_id] - success = True - else: - success, libre_device = api.get_device_info(device_id) - - if not success or not libre_device: - error_msg = f"Failed to retrieve device {device_id} from LibreNMS" - failed_list.append({"device_id": device_id, "error": error_msg}) - if job and job.logger: - job.logger.error(error_msg) - else: - logger.error(error_msg) - continue - - validation = validate_device_for_import(libre_device, api=api) - - # Build manual mappings from validation + any provided overrides - device_mappings = {} - - # Get site and device_type from validation - if validation["site"].get("found") and validation["site"].get("site"): - device_mappings["site_id"] = validation["site"]["site"].id - if validation["device_type"].get("found") and validation["device_type"].get("device_type"): - device_mappings["device_type_id"] = validation["device_type"]["device_type"].id - if validation["platform"].get("found") and validation["platform"].get("platform"): - device_mappings["platform_id"] = validation["platform"]["platform"].id - - # Override with any manual mappings provided for this device - if manual_mappings_per_device and device_id in manual_mappings_per_device: - device_mappings.update(manual_mappings_per_device[device_id]) - - result = import_single_device( - device_id, - server_key=server_key, - sync_options=sync_options, - manual_mappings=device_mappings if device_mappings else None, - libre_device=libre_device, - ) - - if result["success"]: - success_list.append( - { - "device_id": device_id, - "device": result["device"], - "message": result["message"], - } - ) - - # Handle virtual chassis creation for stacks - vc_data = validation.get("virtual_chassis", {}) - if vc_data.get("is_stack", False): - vc_domain = f"librenms-{device_id}" - - # Only create VC if we haven't processed this stack yet - # Add to set BEFORE attempting creation to prevent race condition - if vc_domain not in processed_vc_domains: - processed_vc_domains.add(vc_domain) - try: - vc = create_virtual_chassis_with_members( - result["device"], - vc_data["members"], - libre_device, - ) - vc_created_count += 1 - log_msg = f"Created VC '{vc.name}' during bulk import for device {device_id}" - if job and job.logger: - job.logger.info(log_msg) - else: - logger.info(log_msg) - except Exception as vc_error: - # Remove from set on failure so retry is possible - processed_vc_domains.discard(vc_domain) - warn_msg = f"Failed to create VC for device {device_id}: {vc_error}" - if job and job.logger: - job.logger.warning(warn_msg) - else: - logger.warning(warn_msg) - # Don't fail the import, just log the warning - - elif result.get("device"): # Device exists - skipped_list.append({"device_id": device_id, "reason": result["error"]}) - else: # Failed to import - failed_list.append({"device_id": device_id, "error": result["error"]}) - if job and job.logger: - job.logger.error(f"Failed to import device {device_id}: {result['error']}") - - except Exception as e: - error_msg = f"Unexpected error importing device {device_id}: {str(e)}" - if job and job.logger: - job.logger.error(error_msg, exc_info=True) - else: - logger.exception(f"Unexpected error importing device {device_id}") - failed_list.append({"device_id": device_id, "error": str(e)}) - - return { - "total": total, - "success": success_list, - "failed": failed_list, - "skipped": skipped_list, - "virtual_chassis_created": vc_created_count, - } - - -def bulk_import_devices( - device_ids: List[int], - server_key: str = None, - sync_options: dict = None, - manual_mappings_per_device: dict = None, - libre_devices_cache: dict = None, - user=None, -) -> dict: - """ - Import multiple LibreNMS devices to NetBox (synchronous). - - This is the public API for synchronous imports. For background job usage, - use bulk_import_devices_shared() with a job context. - - Args: - device_ids: List of LibreNMS device IDs to import - server_key: LibreNMS server configuration key - sync_options: Sync options to apply to all devices - manual_mappings_per_device: Dict mapping device_id to manual_mappings dict - Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} - libre_devices_cache: Optional dict mapping device_id to pre-fetched device data - to avoid redundant API calls. Example: {123: {...device_data...}} - user: User performing the import (for permission checks) - - Returns: - dict: Bulk import result with structure: - { - 'total': int, - 'success': List[dict], # Successfully imported devices - 'failed': List[dict], # Failed imports with errors - 'skipped': List[dict], # Skipped devices (already exist, etc.) - 'virtual_chassis_created': int # Number of VCs created - } - - Raises: - PermissionDenied: If user lacks required permissions - """ - return bulk_import_devices_shared( - device_ids=device_ids, - server_key=server_key, - sync_options=sync_options, - manual_mappings_per_device=manual_mappings_per_device, - libre_devices_cache=libre_devices_cache, - job=None, # No job context for synchronous imports - user=user, - ) - - -def get_librenms_device_by_id(api: LibreNMSAPI, device_id: int) -> dict: - """ - Retrieve a single device from LibreNMS by ID. - - Args: - api: LibreNMSAPI instance - device_id: LibreNMS device ID - - Returns: - Device dictionary or None if not found - """ - try: - # Use the dedicated API endpoint to get device by ID - success, device = api.get_device_info(device_id) - if success and device: - return device - - logger.warning(f"Device {device_id} not found in LibreNMS") - return None - except Exception as e: - logger.exception(f"Failed to get device {device_id} from LibreNMS: {e}") - return None - - -def fetch_device_with_cache( - device_id: int, - api: LibreNMSAPI, - server_key: str = None, - libre_devices_cache: dict = None, -) -> dict | None: - """ - Fetch LibreNMS device from cache or API with automatic caching. - - Checks three sources in order: - 1. Pre-fetched cache dict (if provided) - 2. Django cache (Redis/memory) - 3. LibreNMS API (caches result for future use) - - This function consolidates the device fetching pattern used throughout - the import workflow, eliminating code duplication. - - Args: - device_id: LibreNMS device ID to fetch - api: LibreNMSAPI instance for fallback API calls - server_key: Optional server key for multi-server setups (defaults to api.server_key) - libre_devices_cache: Optional pre-fetched device cache dict - - Returns: - Device dict from LibreNMS, or None if not found - - Example: - >>> # Simple usage - >>> libre_device = fetch_device_with_cache(123, api) - >>> if libre_device: - ... print(libre_device['hostname']) - >>> - >>> # With pre-fetched cache dict - >>> cache_dict = {123: {...}, 456: {...}} - >>> libre_device = fetch_device_with_cache(123, api, libre_devices_cache=cache_dict) - """ - # Check pre-fetched cache dict first (fastest) - if libre_devices_cache and device_id in libre_devices_cache: - return libre_devices_cache[device_id] - - # Check Django cache - cache_key = get_import_device_cache_key(device_id, server_key or api.server_key) - libre_device = cache.get(cache_key) - - if not libre_device: - # Fallback to API fetch - libre_device = get_librenms_device_by_id(api, device_id) - if libre_device: - # Cache for future use - cache.set(cache_key, libre_device, timeout=api.cache_timeout) - - return libre_device - - -def create_vm_from_librenms(libre_device: dict, validation: dict, use_sysname: bool = True, role=None): - """ - Create a NetBox VirtualMachine from LibreNMS device data. - - Args: - libre_device: Device data from LibreNMS - validation: Validation result from validate_device_for_import with import_as_vm=True - use_sysname: If True, prefer sysName; if False, use hostname - role: Optional DeviceRole to assign to the VM - - Returns: - Created VirtualMachine instance - - Raises: - Exception if VM cannot be created - """ - from virtualization.models import VirtualMachine - - if not validation["can_import"]: - raise ValueError(f"VM cannot be imported: {', '.join(validation['issues'])}") - - # Extract matched objects from validation - cluster = validation["cluster"]["cluster"] - platform = validation["platform"].get("platform") - - # Determine VM name - use pre-computed name if available (handles strip_domain) - vm_name = libre_device.get("_computed_name") - if not vm_name: - vm_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=False, - device_id=libre_device.get("device_id"), - ) - - # Generate import timestamp comment - import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") - - # Create the VM with librenms_id custom field - vm = VirtualMachine.objects.create( - name=vm_name, - cluster=cluster, - role=role, # Optional VM role - platform=platform, - comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - custom_field_data={"librenms_id": int(libre_device["device_id"])}, - ) - - logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") - return vm - - -def bulk_import_vms( - vm_imports: dict[int, dict[str, int]], - api: LibreNMSAPI, - sync_options: dict = None, - libre_devices_cache: dict = None, - job=None, - user=None, -) -> dict: - """ - Import multiple LibreNMS devices as VMs in NetBox. - - Handles validation, cluster/role assignment, name determination, - and VM creation. Supports both synchronous and background job execution. - - This function consolidates VM import logic that was previously duplicated - in BulkImportDevicesView and ImportDevicesJob, ensuring consistent behavior - across synchronous and background import paths. - - Args: - vm_imports: Dict mapping device_id to {"cluster_id": int, "device_role_id": int} - api: LibreNMSAPI instance for device fetching - sync_options: Optional dict with use_sysname, strip_domain settings - libre_devices_cache: Optional pre-fetched device data cache - job: Optional JobRunner instance for background job logging/cancellation - user: User performing the import (for permission checks). If job is provided, - user is extracted from job.job.user if not explicitly passed. - - Returns: - Dict with keys: - - success: List of {"device_id": int, "device": VM, "message": str} - - failed: List of {"device_id": int, "error": str} - - skipped: List of {"device_id": int, "reason": str} - - Raises: - PermissionDenied: If user lacks required permissions - - Example: - >>> # Synchronous import from view - >>> vm_imports = {123: {"cluster_id": 5, "device_role_id": 2}} - >>> result = bulk_import_vms(vm_imports, api, sync_options, user=request.user) - >>> print(f"Created {len(result['success'])} VMs") - >>> - >>> # Background job import - >>> result = bulk_import_vms(vm_imports, api, sync_options, cache, job=self) - """ - from netbox_librenms_plugin.import_validation_helpers import ( - apply_cluster_to_validation, - apply_role_to_validation, - ) - - # Extract user from job if not explicitly provided - if user is None and job is not None: - user = getattr(job.job, "user", None) - - # Check permissions at start of bulk operation - require_permissions(user, ["virtualization.add_virtualmachine"], "import VMs") - - result = {"success": [], "failed": [], "skipped": []} - vm_ids = list(vm_imports.keys()) - - # Use job logger if available, otherwise standard logger - log = job.logger if job else logger - - for idx, vm_id in enumerate(vm_ids, start=1): - # Check for job cancellation every 5 VMs - if job and idx % 5 == 0: - job.job.refresh_from_db() - job_status = job.job.status - status_value = job_status.value if hasattr(job_status, "value") else job_status - if status_value in ("failed", "errored"): - log.warning(f"Job cancelled at VM {idx} of {len(vm_ids)}") - break - log.info(f"Imported VM {idx} of {len(vm_ids)}") - - try: - # Fetch device data (uses cache helper) - libre_device = fetch_device_with_cache(vm_id, api, api.server_key, libre_devices_cache) - - if not libre_device: - result["failed"].append( - { - "device_id": vm_id, - "error": f"Device {vm_id} not found in LibreNMS", - } - ) - log.error(f"Device {vm_id} not found in LibreNMS") - continue - - # Validate as VM - validation = validate_device_for_import(libre_device, import_as_vm=True, api=api) - - # Check if VM already exists - if validation.get("existing_device"): - result["skipped"].append( - { - "device_id": vm_id, - "reason": f"VM already exists: {validation['existing_device'].name}", - } - ) - log.info(f"VM already exists: {validation['existing_device'].name}") - continue - - # Apply manual cluster and role selections - vm_mappings = vm_imports[vm_id] - cluster_id = vm_mappings.get("cluster_id") - role_id = vm_mappings.get("device_role_id") - - if cluster_id: - cluster = Cluster.objects.filter(id=cluster_id).first() - if cluster: - apply_cluster_to_validation(validation, cluster) - - role = None - if role_id: - role = DeviceRole.objects.filter(id=role_id).first() - if role: - apply_role_to_validation(validation, role, is_vm=True) - - # Determine VM name - use_sysname = sync_options.get("use_sysname", True) if sync_options else True - strip_domain = sync_options.get("strip_domain", False) if sync_options else False - - vm_name = _determine_device_name( - libre_device, - use_sysname=use_sysname, - strip_domain=strip_domain, - device_id=vm_id, - ) - - # Update validation with computed name - libre_device["_computed_name"] = vm_name - - # Create VM - vm = create_vm_from_librenms(libre_device, validation, use_sysname=use_sysname, role=role) - - result["success"].append( - { - "device_id": vm_id, - "device": vm, - "message": f"VM {vm.name} created successfully", - } - ) - log.info(f"Successfully imported VM {vm.name} (ID: {vm_id})") - - except Exception as vm_error: - log.error(f"Failed to import VM {vm_id}: {vm_error}", exc_info=True) - result["failed"].append({"device_id": vm_id, "error": str(vm_error)}) - - return result - - -def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> dict: - """ - Detect if device is a stack/Virtual Chassis by analyzing ENTITY-MIB inventory. - Vendor-agnostic using standard hierarchical structure. - - Args: - api: LibreNMSAPI instance - device_id: LibreNMS device ID - - Returns: - dict with structure: - { - 'is_stack': bool, - 'member_count': int, - 'members': [ - { - 'serial': str, - 'position': int, - 'model': str, - 'name': str, - 'index': int, - 'description': str, - 'suggested_name': str # Generated using master device name - } - ] - } - Returns None if not a stack or detection fails. - - Detection Logic: - 1. Check root level (entPhysicalContainedIn=0) for parent container - 2. Find parent index (entPhysicalClass='stack' or 'chassis') - 3. Get children chassis at that parent's index - 4. If multiple chassis found → Stack detected - """ - try: - # Get the master device info to use for naming - success, device_info = api.get_device_info(device_id) - master_name = None - if success and device_info: - master_name = device_info.get("sysName") or device_info.get("hostname") - - # Step 1: Get root level items - success, root_items = api.get_inventory_filtered(device_id, ent_physical_contained_in=0) - - if not success or not root_items: - logger.debug(f"No root inventory items found for device {device_id}") - return None - - # Step 2: Find parent container index - # Could be class="stack" or the main "chassis" - parent_index = None - for item in root_items: - item_class = item.get("entPhysicalClass") - if item_class in ["stack", "chassis"]: - parent_index = item.get("entPhysicalIndex") - logger.debug(f"VC detection: Found parent container at index {parent_index} for device {device_id}") - break - - if not parent_index: - return None - - # Step 3: Get children chassis at next level - success, child_items = api.get_inventory_filtered( - device_id, - ent_physical_class="chassis", - ent_physical_contained_in=parent_index, - ) - - if not success: - return None - - # Filter for chassis only (in case API filter didn't work) - chassis_items = [item for item in (child_items or []) if item.get("entPhysicalClass") == "chassis"] - - # Step 4: Multiple chassis = stack - if len(chassis_items) <= 1: - return None - - # Step 5: Extract member info - members = [] - for idx, chassis in enumerate(chassis_items): - raw_position = chassis.get("entPhysicalParentRelPos", idx) - try: - position = int(raw_position) - except (TypeError, ValueError): - position = idx - member_data = { - "serial": chassis.get("entPhysicalSerialNum", ""), - "position": position, - "model": chassis.get("entPhysicalModelName", ""), - "name": chassis.get("entPhysicalName", ""), - "index": chassis.get("entPhysicalIndex"), - "description": chassis.get("entPhysicalDescr", ""), - } - - # Generate suggested name if we have master name - if master_name: - member_data["suggested_name"] = _generate_vc_member_name(master_name, position + 1) - else: - member_data["suggested_name"] = f"Member-{position + 1}" - - members.append(member_data) - - # Sort by position - members.sort(key=lambda m: m["position"]) - - logger.info(f"Detected stack with {len(members)} members for device {device_id}") - - return {"is_stack": True, "member_count": len(members), "members": members} - - except Exception as e: - logger.exception(f"Error detecting virtual chassis for device {device_id}: {e}") - return None - - -def _generate_vc_member_name(master_name: str, position: int, serial: str = None) -> str: - """ - Generate name for VC member device using configured pattern from settings. - - Args: - master_name: Name of the master/primary device - position: VC position number - serial: Optional serial number of the member device - - Returns: - Generated member device name - - Examples: - pattern="-M{position}" → "switch01-M2" - pattern=" ({position})" → "switch01 (2)" - pattern="-SW{position}" → "switch01-SW2" - pattern=" [{serial}]" → "switch01 [ABC123]" - """ - # Import here to avoid circular dependency - from .models import LibreNMSSettings - - # Get pattern from settings with fallback to default - try: - settings = LibreNMSSettings.objects.first() - pattern = settings.vc_member_name_pattern if settings else "-M{position}" - except Exception as e: - logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") - pattern = "-M{position}" - - # Prepare format variables - format_vars = { - "master_name": master_name, - "position": position, - "serial": serial or "", - } - - # Apply pattern - pattern should be suffix/prefix, not full name - try: - formatted_suffix = pattern.format(**format_vars) - return f"{master_name}{formatted_suffix}" - except KeyError as e: - logger.error(f"Invalid placeholder in VC naming pattern '{pattern}': {e}. Using default.") - return f"{master_name}-M{position}" - - -def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: - """ - Regenerate suggested VC member names using the actual master device name. - - This ensures preview shows accurate names after use_sysname and strip_domain - are applied to the master device name. - - Args: - vc_data: Virtual chassis detection data dict - master_name: The actual name that will be used for master device in NetBox - - Returns: - Updated vc_data dict with corrected suggested_name for each member - """ - if not vc_data or not vc_data.get("is_stack"): - return vc_data - - for idx, member in enumerate(vc_data.get("members", [])): - raw_position = member.get("position", idx) - try: - base_position = int(raw_position) - except (TypeError, ValueError): - base_position = idx - position = base_position + 1 # Convert to 1-based position - member["position"] = base_position - member["suggested_name"] = _generate_vc_member_name(master_name, position, serial=member.get("serial")) - - return vc_data - - -def create_virtual_chassis_with_members(master_device: Device, members_info: list, libre_device: dict): - """ - Create Virtual Chassis and member devices from detection info. - - This function creates a NetBox VirtualChassis with the master device - and all detected member devices, wrapped in a transaction for safety. - - Args: - master_device: The imported device (becomes VC master) - members_info: List of member dicts from VC detection - libre_device: Original LibreNMS device data - - Returns: - VirtualChassis: The created virtual chassis instance - - Raises: - ValidationError: If member count validation fails - IntegrityError: If duplicate serials/names are detected - Exception: For other creation errors - - Example members_info: - [ - {'serial': 'ABC123', 'position': 0, 'model': 'C9300-48U', 'name': 'Switch 1'}, - {'serial': 'ABC124', 'position': 1, 'model': 'C9300-48U', 'name': 'Switch 2'} - ] - """ - - # Store original master device state for rollback - original_master_name = master_device.name - original_vc = master_device.virtual_chassis - original_vc_position = master_device.vc_position - - try: - with transaction.atomic(): - # Rename master device to include position 1 pattern - master_device_new_name = _generate_vc_member_name(original_master_name, 1, serial=master_device.serial) - - # Check if renamed master conflicts with existing device - if Device.objects.filter(name=master_device_new_name).exclude(pk=master_device.pk).exists(): - logger.warning( - f"Cannot rename master to '{master_device_new_name}' - name already exists. " - f"Keeping original name '{original_master_name}'" - ) - master_base_name = original_master_name - else: - master_device.name = master_device_new_name - master_base_name = original_master_name - - # Create VC using original base name - vc_name = master_base_name - vc = VirtualChassis.objects.create( - name=vc_name, - master=master_device, - domain=f"librenms-{libre_device['device_id']}", - ) - - # Update master device - master_device.virtual_chassis = vc - master_device.vc_position = 1 # Master is position 1 - master_device.save() - - # Create member devices for remaining positions - position = 2 # Start at 2 (master is 1) - members_created = 0 - - for member in members_info: - # Skip if this is the master's serial - if member.get("serial") == master_device.serial: - continue - - serial = member.get("serial") - - member_rack = master_device.rack - member_location = master_device.location or ( - member_rack.location if member_rack and member_rack.location else None - ) - - # Check for duplicate serial - if serial and Device.objects.filter(serial=serial).exists(): - logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") - continue - - member_name = _generate_vc_member_name(master_base_name, position, serial=serial) - - # Check for duplicate name - if Device.objects.filter(name=member_name).exists(): - logger.warning(f"Device with name '{member_name}' already exists, skipping VC member creation") - continue - - Device.objects.create( - name=member_name, - device_type=master_device.device_type, - role=master_device.role, - site=master_device.site, - location=member_location, - rack=member_rack, - platform=master_device.platform, - serial=serial, - virtual_chassis=vc, - vc_position=position, - comments=f"VC member (LibreNMS: {member.get('name', 'Unknown')})\n" - f"Auto-created from stack inventory", - ) - members_created += 1 - position += 1 - - # Validate member count - expected_members = len([m for m in members_info if m.get("serial") != master_device.serial]) - if members_created < expected_members: - logger.warning( - f"Created {members_created} members but expected {expected_members}. " - "Some members may have been skipped due to duplicates." - ) - - logger.info( - f"Created Virtual Chassis '{vc.name}' with {vc.members.count()} total members " - f"(1 master + {members_created} additional)" - ) - - return vc - - except Exception as e: - # Rollback master device to original state - logger.error( - f"Virtual Chassis creation failed for device {master_device.name}: {e}. Rolling back master device changes." - ) - master_device.name = original_master_name - master_device.virtual_chassis = original_vc - master_device.vc_position = original_vc_position - master_device.save() - raise - - -def process_device_filters( - api: LibreNMSAPI, - filters: dict, - vc_detection_enabled: bool, - clear_cache: bool, - show_disabled: bool, - exclude_existing: bool = False, - job=None, - request=None, - return_cache_status: bool = False, -) -> List[dict] | tuple[List[dict], bool]: - """ - Process LibreNMS device filters and return validated devices. - - Shared function used by both synchronous view and background job processing. - Fetches devices, optionally pre-warms VC cache, validates each device, and - caches results for HTMX row updates. - - Args: - api: LibreNMS API client instance - filters: Filter dict with location, type, os, hostname, sysname, hardware keys - vc_detection_enabled: Whether to detect virtual chassis - clear_cache: Whether to force cache refresh - show_disabled: Whether to include disabled devices - exclude_existing: Whether to exclude devices that already exist in NetBox - job: Optional JobRunner instance for logging job events - request: Optional Django request for client disconnect detection (synchronous only) - return_cache_status: When True, returns (devices, from_cache) tuple - - Returns: - List[dict]: Validated devices with _validation key, or tuple of (devices, from_cache) - if return_cache_status is True. from_cache=True means data was loaded from existing - cache; from_cache=False means data was just fetched from LibreNMS. - """ - # Fetch devices from LibreNMS - if job: - job.logger.info(f"Fetching devices with filters: {filters}") - else: - logger.info(f"Fetching devices with filters: {filters}") - - # Always get cache status internally, even if not returning it - # We need it to determine if metadata should be updated - libre_devices, from_cache = get_librenms_devices_for_import( - api, - filters=filters, - force_refresh=clear_cache, - return_cache_status=True, - ) - - # Filter out disabled devices if requested - if not show_disabled: - libre_devices = [d for d in libre_devices if d.get("status") == 1] - - if job: - job.logger.info(f"Found {len(libre_devices)} devices to process") - else: - logger.info(f"Found {len(libre_devices)} devices") - - # Pre-warm VC cache if needed - if vc_detection_enabled and libre_devices: - device_ids = [d["device_id"] for d in libre_devices] - if job: - job.logger.info( - f"Pre-fetching virtual chassis data for {len(device_ids)} devices. This may take some time..." - ) - else: - logger.info(f"Pre-fetching VC data for {len(device_ids)} devices") - - try: - prefetch_vc_data_for_devices(api, device_ids, force_refresh=clear_cache) - if job: - job.logger.info("Virtual chassis data pre-fetch completed") - except (BrokenPipeError, ConnectionError, IOError) as e: - if request: - logger.info(f"Client disconnected during VC prefetch: {e}") - return [] - raise - - # Validate each device - validated_devices = [] - total = len(libre_devices) - api_for_validation = api if vc_detection_enabled else None - - if job: - job.logger.info(f"Starting validation of {total} devices") - # Initial check if job was already terminated before we even started - try: - from django_rq import get_queue - from rq.job import Job as RQJob - - queue = get_queue("default") - rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) - - if rq_job.is_failed or rq_job.is_stopped: - job.logger.warning("Job was already stopped before validation started") - return [] - except Exception: - # Fall back to DB check if RQ check fails - job.job.refresh_from_db() - if job.job.status == JobStatusChoices.STATUS_FAILED: - job.logger.warning("Job was stopped before validation started") - return [] - else: - logger.info(f"Validating {total} devices") - - for idx, device in enumerate(libre_devices, 1): - # Check for job termination or client disconnect periodically - if idx % 5 == 0 or idx == 1: # Check more frequently (every 5 devices + first device) - if job: - # Check if job was terminated via stop API - # CRITICAL: Check the RQ job status in Redis, not just the DB model - # NetBox's stop endpoint marks the RQ job as failed in Redis - try: - from django_rq import get_queue - from rq.job import Job as RQJob - - queue = get_queue("default") - rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) - - # Check if RQ job is in a stopped state - if rq_job.is_failed or rq_job.is_stopped: - job.logger.info( - f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." - ) - return [] - except Exception: - # If we can't check RQ status, fall back to DB status check - job.job.refresh_from_db() - if job.job.status == JobStatusChoices.STATUS_FAILED: - job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") - return [] - elif request: - # Check for client disconnect - try: - if hasattr(request, "META") and request.META.get("wsgi.input"): - pass - except (BrokenPipeError, ConnectionError, IOError): - logger.info(f"Client disconnected during validation at device {idx}") - return [] - - # Drop any cached validation/meta keys before recomputing - device.pop("_validation", None) - - # Generate shared cache key for this validated device - device_id = device["device_id"] - cache_key = get_validated_device_cache_key( - server_key=api.server_key, - filters=filters, - device_id=device_id, - vc_enabled=vc_detection_enabled, - ) - - # Check if we already have cached validation for this device - # (only if not forcing refresh) - if not clear_cache: - cached_device = cache.get(cache_key) - if cached_device: - # Use cached validation - device["_validation"] = cached_device["_validation"] - - # Apply exclude_existing filter if enabled - if exclude_existing: - validation = device["_validation"] - if validation["existing_device"]: - continue - - validated_devices.append(device) - continue - - # Not in cache or forcing refresh - validate now - try: - validation = validate_device_for_import( - device, - api=api_for_validation, - include_vc_detection=vc_detection_enabled, - force_vc_refresh=clear_cache, - ) - except (BrokenPipeError, ConnectionError, IOError) as e: - if request: - logger.info(f"Client disconnected during device validation: {e}") - return [] - raise - - # Set VC detection metadata - if not vc_detection_enabled: - validation["virtual_chassis"] = empty_virtual_chassis_data() - - # Apply exclude_existing filter if enabled - if exclude_existing and validation["existing_device"]: - continue - - device["_validation"] = validation - validated_devices.append(device) - - # Cache with TWO keys for different purposes: - # 1. Complex key (with filter context) - for full validated device with all metadata - cache.set(cache_key, device, timeout=api.cache_timeout) - - # 2. Simple key (device ID only) - for quick device data lookup by role/rack updates - # This avoids redundant API calls when user interacts with dropdowns - simple_cache_key = get_import_device_cache_key(device_id, api.server_key) - # Cache just the raw device data (not the full validation result) - # This is what get_validated_device_with_selections() expects - device_data_only = {k: v for k, v in device.items() if k != "_validation"} - cache.set(simple_cache_key, device_data_only, timeout=api.cache_timeout) - - # Store cache metadata (timestamp) for all filter operations - # This enables countdown display regardless of background job vs synchronous execution - # Always store metadata when we have validated devices, even if from_cache - # This ensures metadata is available for countdown display - if validated_devices: - from datetime import datetime, timezone - - cache_metadata_key = get_cache_metadata_key( - server_key=api.server_key, filters=filters, vc_enabled=vc_detection_enabled - ) - - # Check if metadata already exists to preserve original timestamp - # BUT: if clear_cache was requested or data came fresh from LibreNMS, update it - existing_metadata = cache.get(cache_metadata_key) - should_update = clear_cache or not from_cache - - if existing_metadata and not should_update: - # Metadata exists and cache wasn't cleared, keep using it (preserves original cache time) - pass - else: - # No metadata exists, OR cache was cleared, OR fresh data - create/update it now - cache_metadata = { - "cached_at": datetime.now(timezone.utc).isoformat(), - "cache_timeout": api.cache_timeout, - "filters": filters, - "vc_enabled": vc_detection_enabled, - "device_count": len(validated_devices), - } - cache.set(cache_metadata_key, cache_metadata, timeout=api.cache_timeout) - - # Maintain cache index for this server to enable listing active searches - cache_index_key = f"librenms_cache_index_{api.server_key}" - cache_index = cache.get(cache_index_key, []) - # Add this cache key if not already in index - if cache_metadata_key not in cache_index: - cache_index.append(cache_metadata_key) - # Store index with same timeout as the metadata - cache.set(cache_index_key, cache_index, timeout=api.cache_timeout) - - if job: - if exclude_existing: - filtered_count = total - len(validated_devices) - job.logger.info( - f"Validation complete: {len(validated_devices)} devices passed filter, " - f"{filtered_count} filtered out (existing devices excluded)" - ) - else: - job.logger.info(f"Validation complete: {len(validated_devices)} devices ready for import") - else: - logger.info(f"Processed {len(validated_devices)} validated devices") - - if return_cache_status: - return validated_devices, from_cache - return validated_devices diff --git a/netbox_librenms_plugin/import_utils/__init__.py b/netbox_librenms_plugin/import_utils/__init__.py new file mode 100644 index 0000000000..b7a08fef59 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/__init__.py @@ -0,0 +1,52 @@ +""" +Utilities for importing devices from LibreNMS to NetBox. + +This package provides functions for: +- Validating LibreNMS devices for import +- Retrieving filtered LibreNMS devices +- Importing single and multiple devices +- Smart matching of NetBox objects +- Permission checking for import operations +- Virtual chassis detection and creation + +All imports below are intentional re-exports so that existing callers +can continue using ``from netbox_librenms_plugin.import_utils import X``. +The F401 suppressions prevent linters from flagging them as unused. +""" + +from .bulk_import import ( # noqa: F401 + bulk_import_devices, + bulk_import_devices_shared, + process_device_filters, +) +from .cache import ( # noqa: F401 + get_active_cached_searches, + get_cache_metadata_key, + get_import_device_cache_key, + get_validated_device_cache_key, +) +from .device_operations import ( # noqa: F401 + _determine_device_name, + fetch_device_with_cache, + get_librenms_device_by_id, + import_single_device, + validate_device_for_import, +) +from .filters import ( # noqa: F401 + _apply_client_filters, + get_device_count_for_filters, + get_librenms_devices_for_import, +) +from .permissions import check_user_permissions, require_permissions # noqa: F401 +from .virtual_chassis import ( # noqa: F401 + _clone_virtual_chassis_data, + _generate_vc_member_name, + _vc_cache_key, + create_virtual_chassis_with_members, + detect_virtual_chassis_from_inventory, + empty_virtual_chassis_data, + get_virtual_chassis_data, + prefetch_vc_data_for_devices, + update_vc_member_suggested_names, +) +from .vm_operations import bulk_import_vms, create_vm_from_librenms # noqa: F401 diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py new file mode 100644 index 0000000000..8c680b864a --- /dev/null +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -0,0 +1,659 @@ +"""Bulk import orchestration for devices and filter processing.""" + +import logging +from typing import List + +from core.choices import JobStatusChoices +from django.core.cache import cache + +from ..librenms_api import LibreNMSAPI +from ..utils import find_by_librenms_id +from .cache import get_cache_metadata_key, get_import_device_cache_key, get_validated_device_cache_key +from .device_operations import import_single_device, validate_device_for_import +from .filters import get_librenms_devices_for_import +from .permissions import require_permissions +from .virtual_chassis import ( + create_virtual_chassis_with_members, + empty_virtual_chassis_data, + prefetch_vc_data_for_devices, +) + +logger = logging.getLogger(__name__) + + +def _empty_return(return_cache_status: bool): + """Centralised empty-result return value for process_device_filters.""" + return ([], False) if return_cache_status else [] + + +def bulk_import_devices_shared( + device_ids: List[int], + server_key: str = None, + sync_options: dict = None, + manual_mappings_per_device: dict = None, + libre_devices_cache: dict = None, + job=None, + user=None, +) -> dict: + """ + Shared function for importing multiple LibreNMS devices to NetBox. + + Used by both synchronous imports and background jobs. Handles per-device error + collection and optional progress logging when job context is provided. + + Args: + device_ids: List of LibreNMS device IDs to import + server_key: LibreNMS server configuration key + sync_options: Sync options to apply to all devices + manual_mappings_per_device: Dict mapping device_id to manual_mappings dict + Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} + libre_devices_cache: Optional dict mapping device_id to pre-fetched device data + to avoid redundant API calls. Example: {123: {...device_data...}} + job: Optional JobRunner instance for progress logging and cancellation checks + user: User performing the import (for permission checks). If job is provided, + user is extracted from job.job.user if not explicitly passed. + + Returns: + dict: Bulk import result with structure: + { + 'total': int, + 'success': List[dict], # Successfully imported devices + 'failed': List[dict], # Failed imports with errors + 'skipped': List[dict], # Skipped devices (already exist, etc.) + 'virtual_chassis_created': int # Number of VCs created + } + + Raises: + PermissionDenied: If user lacks required permissions + + Example: + >>> # Synchronous usage + >>> result = bulk_import_devices_shared([1, 2, 3, 4, 5], user=request.user) + >>> # Background job usage + >>> result = bulk_import_devices_shared([1, 2, 3], job=self) + """ + # Extract user from job if not explicitly provided + if user is None and job is not None: + user = getattr(job.job, "user", None) + + # Check permissions at start of bulk operation + required_perms = [ + "dcim.add_device", + "dcim.add_interface", + "dcim.add_virtualchassis", + ] + require_permissions(user, required_perms, "import devices") + + total = len(device_ids) + success_list = [] + failed_list = [] + skipped_list = [] + vc_created_count = 0 + processed_vc_domains = set() # Track VCs already created by domain + + # Initialize API client once for all devices to avoid repeated config parsing + api = LibreNMSAPI(server_key=server_key) + + for idx, device_id in enumerate(device_ids, start=1): + # Check for job cancellation on first iteration and every 5th thereafter. + # Check RQ/Redis state first (reflects stop API immediately); fall back to DB. + if job and (idx == 1 or idx % 5 == 0): + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + if rq_job.is_failed or rq_job.is_stopped: + if job.logger: + job.logger.warning( + f"Import job stopped at device {idx} of {total} (RQ status: {rq_job.get_status()})" + ) + else: + logger.warning(f"Import cancelled at device {idx} of {total}") + break + except Exception: + # Fall back to DB check if RQ is unavailable + job.job.refresh_from_db() + job_status = job.job.status + status_value = job_status.value if hasattr(job_status, "value") else job_status + if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): + if job.logger: + job.logger.warning(f"Import job cancelled at device {idx} of {total}") + else: + logger.warning(f"Import cancelled at device {idx} of {total}") + break + + try: + # Use cached device data if available to avoid redundant API calls + if libre_devices_cache and device_id in libre_devices_cache: + libre_device = libre_devices_cache[device_id] + success = True + else: + success, libre_device = api.get_device_info(device_id) + + if not success or not libre_device: + error_msg = f"Failed to retrieve device {device_id} from LibreNMS" + failed_list.append({"device_id": device_id, "error": error_msg}) + if job and job.logger: + job.logger.error(error_msg) + else: + logger.error(error_msg) + continue + + use_sysname_opt = sync_options.get("use_sysname", True) if sync_options else True + strip_domain_opt = sync_options.get("strip_domain", False) if sync_options else False + validation = validate_device_for_import( + libre_device, + api=api, + use_sysname=use_sysname_opt, + strip_domain=strip_domain_opt, + server_key=api.server_key, + ) + + # Build manual mappings from validation + any provided overrides + device_mappings = {} + + # Get site and device_type from validation + if validation["site"].get("found") and validation["site"].get("site"): + device_mappings["site_id"] = validation["site"]["site"].id + if validation["device_type"].get("found") and validation["device_type"].get("device_type"): + device_mappings["device_type_id"] = validation["device_type"]["device_type"].id + if validation["platform"].get("found") and validation["platform"].get("platform"): + device_mappings["platform_id"] = validation["platform"]["platform"].id + + # Override with any manual mappings provided for this device + if manual_mappings_per_device and device_id in manual_mappings_per_device: + device_mappings.update(manual_mappings_per_device[device_id]) + + result = import_single_device( + device_id, + server_key=api.server_key, # use resolved key, not raw parameter (may be None) + sync_options=sync_options, + manual_mappings=device_mappings if device_mappings else None, + libre_device=libre_device, + ) + + if result["success"]: + success_list.append( + { + "device_id": device_id, + "device": result["device"], + "message": result["message"], + } + ) + # Log progress after each successful import + if job and job.logger: + job.logger.info(f"Imported device {idx} of {total}") + + # Handle virtual chassis creation for stacks + vc_data = validation.get("virtual_chassis", {}) + if vc_data.get("is_stack", False): + # Derive a stack-level dedup key from member serials so that all + # LibreNMS devices belonging to the same physical stack (e.g. each + # switch in a stacked chassis that appears as a separate device in + # LibreNMS) share the same key and VC creation is triggered only once. + # Fall back to device_id when no member serials are available. + member_serials = sorted(m.get("serial") for m in vc_data.get("members", []) if m.get("serial")) + vc_domain = ( + f"librenms-stack-{','.join(member_serials)}" if member_serials else f"librenms-{device_id}" + ) + + # Only create VC if we haven't processed this stack yet + # Add to set BEFORE attempting creation to prevent race condition + if vc_domain not in processed_vc_domains: + processed_vc_domains.add(vc_domain) + try: + vc = create_virtual_chassis_with_members( + result["device"], + vc_data["members"], + libre_device, + ) + vc_created_count += 1 + log_msg = f"Created VC '{vc.name}' during bulk import for device {device_id}" + if job and job.logger: + job.logger.info(log_msg) + else: + logger.info(log_msg) + except Exception as vc_error: + # Remove from set on failure so retry is possible + processed_vc_domains.discard(vc_domain) + warn_msg = f"Failed to create VC for device {device_id}: {vc_error}" + if job and job.logger: + job.logger.warning(warn_msg) + else: + logger.warning(warn_msg) + # Don't fail the import, just log the warning + + elif result.get("device"): # Device exists + skipped_list.append({"device_id": device_id, "reason": result["error"]}) + else: # Failed to import + failed_list.append({"device_id": device_id, "error": result["error"]}) + if job and job.logger: + job.logger.error(f"Failed to import device {device_id}: {result['error']}") + + except Exception as e: + error_msg = f"Unexpected error importing device {device_id}: {str(e)}" + if job and job.logger: + job.logger.error(error_msg, exc_info=True) + else: + logger.exception(f"Unexpected error importing device {device_id}") + failed_list.append({"device_id": device_id, "error": str(e)}) + + return { + "total": total, + "success": success_list, + "failed": failed_list, + "skipped": skipped_list, + "virtual_chassis_created": vc_created_count, + } + + +def bulk_import_devices( + device_ids: List[int], + server_key: str = None, + sync_options: dict = None, + manual_mappings_per_device: dict = None, + libre_devices_cache: dict = None, + user=None, +) -> dict: + """ + Import multiple LibreNMS devices to NetBox (synchronous). + + This is the public API for synchronous imports. For background job usage, + use bulk_import_devices_shared() with a job context. + + Args: + device_ids: List of LibreNMS device IDs to import + server_key: LibreNMS server configuration key + sync_options: Sync options to apply to all devices + manual_mappings_per_device: Dict mapping device_id to manual_mappings dict + Example: {1179: {'device_role_id': 5}, 1180: {'device_role_id': 3}} + libre_devices_cache: Optional dict mapping device_id to pre-fetched device data + to avoid redundant API calls. Example: {123: {...device_data...}} + user: User performing the import (for permission checks) + + Returns: + dict: Bulk import result with structure: + { + 'total': int, + 'success': List[dict], # Successfully imported devices + 'failed': List[dict], # Failed imports with errors + 'skipped': List[dict], # Skipped devices (already exist, etc.) + 'virtual_chassis_created': int # Number of VCs created + } + + Raises: + PermissionDenied: If user lacks required permissions + """ + return bulk_import_devices_shared( + device_ids=device_ids, + server_key=server_key, + sync_options=sync_options, + manual_mappings_per_device=manual_mappings_per_device, + libre_devices_cache=libre_devices_cache, + job=None, # No job context for synchronous imports + user=user, + ) + + +def _refresh_existing_device(validation: dict, libre_device: dict = None, server_key: str = "default") -> None: + """Refresh existing_device from DB to pick up changes made in NetBox since caching. + + When existing_device is None (wasn't found at cache time), re-check if the device + was imported since caching by looking up librenms_id or hostname. + """ + existing = validation.get("existing_device") + if existing and hasattr(existing, "pk"): + try: + from dcim.models import Device + from virtualization.models import VirtualMachine + + if validation.get("import_as_vm"): + refreshed = VirtualMachine.objects.filter(pk=existing.pk).first() + else: + refreshed = Device.objects.filter(pk=existing.pk).first() + + if refreshed: + validation["existing_device"] = refreshed + if hasattr(refreshed, "role") and refreshed.role: + validation["device_role"] = {"found": True, "role": refreshed.role} + else: + # Device was deleted since caching — recompute readiness to match + # validate_device_for_import logic. + validation["existing_device"] = None + validation["existing_match_type"] = None + can_import = not bool(validation.get("issues")) + if validation.get("import_as_vm"): + # VMs only require a cluster (site/role not mandatory) + is_ready = can_import and bool(validation.get("cluster", {}).get("found")) + else: + is_ready = ( + can_import + and bool(validation.get("site", {}).get("found")) + and bool(validation.get("device_type", {}).get("found")) + and bool(validation.get("device_role", {}).get("found")) + ) + validation["can_import"] = can_import + validation["is_ready"] = is_ready + except Exception as e: + existing_id = getattr(existing, "pk", "unknown") if existing else "none" + logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") + return + + # existing_device was None at cache time — check if device was imported since + if not libre_device: + return + try: + from dcim.models import Device + from virtualization.models import VirtualMachine + + import_as_vm = validation.get("import_as_vm", False) + Model = VirtualMachine if import_as_vm else Device + + librenms_id = libre_device.get("device_id") + hostname = libre_device.get("hostname", "") + sys_name = libre_device.get("sysName", "") + + new_device = None + match_type = None + + # Check by librenms_id custom field first (JSON multi-server format + legacy) + if librenms_id: + try: + new_device = find_by_librenms_id(Model, int(librenms_id), server_key) + if new_device: + match_type = "librenms_id" + except (ValueError, TypeError): + pass + + # Fall back to hostname match, then sys_name independently + if not new_device and hostname: + new_device = Model.objects.filter(name__iexact=hostname).first() + if new_device: + match_type = "hostname" + if not new_device and sys_name: + new_device = Model.objects.filter(name__iexact=sys_name).first() + if new_device: + match_type = "hostname" + + if new_device: + validation["existing_device"] = new_device + validation["existing_match_type"] = match_type + validation["can_import"] = False + validation["is_ready"] = False + except Exception as e: + logger.error(f"Failed to re-check for imported device: {e}") + + +def process_device_filters( + api: LibreNMSAPI, + filters: dict, + vc_detection_enabled: bool, + clear_cache: bool, + show_disabled: bool, + exclude_existing: bool = False, + job=None, + request=None, + return_cache_status: bool = False, + use_sysname: bool = True, + strip_domain: bool = False, +) -> List[dict] | tuple[List[dict], bool]: + """ + Process LibreNMS device filters and return validated devices. + + Shared function used by both synchronous view and background job processing. + Fetches devices, optionally pre-warms VC cache, validates each device, and + caches results for HTMX row updates. + + Args: + api: LibreNMS API client instance + filters: Filter dict with location, type, os, hostname, sysname, hardware keys + vc_detection_enabled: Whether to detect virtual chassis + clear_cache: Whether to force cache refresh + show_disabled: Whether to include disabled devices + exclude_existing: Whether to exclude devices that already exist in NetBox + job: Optional JobRunner instance for logging job events + request: Optional Django request for client disconnect detection (synchronous only) + return_cache_status: When True, returns (devices, from_cache) tuple + use_sysname: If True, prefer sysName over hostname for device name resolution + strip_domain: If True, strip domain suffix from device name + + Returns: + List[dict]: Validated devices with _validation key, or tuple of (devices, from_cache) + if return_cache_status is True. from_cache=True means data was loaded from existing + cache; from_cache=False means data was just fetched from LibreNMS. + """ + # Fetch devices from LibreNMS + if job: + job.logger.info(f"Fetching devices with filters: {filters}") + else: + logger.info(f"Fetching devices with filters: {filters}") + + # Always get cache status internally, even if not returning it + # We need it to determine if metadata should be updated + libre_devices, from_cache = get_librenms_devices_for_import( + api, + filters=filters, + force_refresh=clear_cache, + return_cache_status=True, + ) + + # Filter out disabled devices if requested. LibreNMS's "disabled" field (1=disabled, + # 0=enabled) reflects manual device disablement; "status" reflects SNMP reachability. + # show_disabled controls the former: hidden when disabled==1, shown regardless of status. + if not show_disabled: + libre_devices = [d for d in libre_devices if int(d.get("disabled", 0)) != 1] + + if job: + job.logger.info(f"Found {len(libre_devices)} devices to process") + else: + logger.info(f"Found {len(libre_devices)} devices") + + # Pre-warm VC cache if needed + if vc_detection_enabled and libre_devices: + device_ids = [d["device_id"] for d in libre_devices] + if job: + job.logger.info( + f"Pre-fetching virtual chassis data for {len(device_ids)} devices. This may take some time..." + ) + else: + logger.info(f"Pre-fetching VC data for {len(device_ids)} devices") + + try: + prefetch_vc_data_for_devices(api, device_ids, force_refresh=clear_cache) + if job: + job.logger.info("Virtual chassis data pre-fetch completed") + except (BrokenPipeError, ConnectionError, IOError) as e: + if request: + logger.info(f"Client disconnected during VC prefetch: {e}") + return _empty_return(return_cache_status) + raise + + # Validate each device + validated_devices = [] + total = len(libre_devices) + api_for_validation = api if vc_detection_enabled else None + + if job: + job.logger.info(f"Starting validation of {total} devices") + # Initial check if job was already terminated before we even started + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + + if rq_job.is_failed or rq_job.is_stopped: + job.logger.warning("Job was already stopped before validation started") + return _empty_return(return_cache_status) + except Exception: + # Fall back to DB check if RQ check fails + job.job.refresh_from_db() + if job.job.status in (JobStatusChoices.STATUS_FAILED, JobStatusChoices.STATUS_ERRORED): + job.logger.warning("Job was stopped before validation started") + return _empty_return(return_cache_status) + else: + logger.info(f"Validating {total} devices") + + for idx, device in enumerate(libre_devices, 1): + # Check for job termination or client disconnect periodically + if idx % 5 == 0 or idx == 1: # Check more frequently (every 5 devices + first device) + if job: + # Check if job was terminated via stop API + # CRITICAL: Check the RQ job status in Redis, not just the DB model + # NetBox's stop endpoint marks the RQ job as failed in Redis + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + + # Check if RQ job is in a stopped state + if rq_job.is_failed or rq_job.is_stopped: + job.logger.info( + f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." + ) + return _empty_return(return_cache_status) + except Exception: + # If we can't check RQ status, fall back to DB status check + job.job.refresh_from_db() + if job.job.status in (JobStatusChoices.STATUS_FAILED, JobStatusChoices.STATUS_ERRORED): + job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") + return _empty_return(return_cache_status) + + # Drop any cached validation/meta keys before recomputing + device.pop("_validation", None) + + # Generate shared cache key for this validated device + device_id = device["device_id"] + cache_key = get_validated_device_cache_key( + server_key=api.server_key, + filters=filters, + device_id=device_id, + vc_enabled=vc_detection_enabled, + use_sysname=use_sysname, + strip_domain=strip_domain, + ) + + # Check if we already have cached validation for this device + # (only if not forcing refresh) + if not clear_cache: + cached_device = cache.get(cache_key) + if cached_device: + # Use cached validation + device["_validation"] = cached_device["_validation"] + + # Refresh existing_device from DB to avoid stale data + # (user may have changed role, name, etc. in NetBox) + _refresh_existing_device(device["_validation"], libre_device=device, server_key=api.server_key) + + # Apply exclude_existing filter if enabled + if exclude_existing: + validation = device["_validation"] + if validation["existing_device"]: + continue + + validated_devices.append(device) + continue + + # Not in cache or forcing refresh - validate now + try: + validation = validate_device_for_import( + device, + api=api_for_validation, + include_vc_detection=vc_detection_enabled, + force_vc_refresh=clear_cache, + server_key=api.server_key, + use_sysname=use_sysname, + strip_domain=strip_domain, + ) + except (BrokenPipeError, ConnectionError, IOError) as e: + if request: + logger.info(f"Client disconnected during device validation: {e}") + return _empty_return(return_cache_status) + raise + + # Set VC detection metadata + if not vc_detection_enabled: + validation["virtual_chassis"] = empty_virtual_chassis_data() + + # Apply exclude_existing filter if enabled + if exclude_existing and validation["existing_device"]: + continue + + device["_validation"] = validation + validated_devices.append(device) + + # Cache with TWO keys for different purposes: + # 1. Complex key (with filter context) - for full validated device with all metadata + cache.set(cache_key, device, timeout=api.cache_timeout) + + # 2. Simple key (device ID only) - for quick device data lookup by role/rack updates + # This avoids redundant API calls when user interacts with dropdowns + simple_cache_key = get_import_device_cache_key(device_id, api.server_key) + # Cache just the raw device data (not the full validation result) + # This is what get_validated_device_with_selections() expects + device_data_only = {k: v for k, v in device.items() if k != "_validation"} + cache.set(simple_cache_key, device_data_only, timeout=api.cache_timeout) + + # Store cache metadata (timestamp) for all filter operations + # This enables countdown display regardless of background job vs synchronous execution + # Always store metadata when we have validated devices, even if from_cache + # This ensures metadata is available for countdown display + if validated_devices: + from datetime import datetime, timezone + + cache_metadata_key = get_cache_metadata_key( + server_key=api.server_key, + filters=filters, + vc_enabled=vc_detection_enabled, + use_sysname=use_sysname, + strip_domain=strip_domain, + ) + + # Check if metadata already exists to preserve original timestamp + # BUT: if clear_cache was requested or data came fresh from LibreNMS, update it + existing_metadata = cache.get(cache_metadata_key) + should_update = clear_cache or not from_cache + + if existing_metadata and not should_update: + # Metadata exists and cache wasn't cleared, keep using it (preserves original cache time) + pass + else: + # No metadata exists, OR cache was cleared, OR fresh data - create/update it now + cache_metadata = { + "cached_at": datetime.now(timezone.utc).isoformat(), + "cache_timeout": api.cache_timeout, + "filters": filters, + "vc_enabled": vc_detection_enabled, + "device_count": len(validated_devices), + } + cache.set(cache_metadata_key, cache_metadata, timeout=api.cache_timeout) + + # Maintain cache index for this server to enable listing active searches + cache_index_key = f"librenms_cache_index_{api.server_key}" + cache_index = cache.get(cache_index_key, []) + # Add this cache key if not already in index + if cache_metadata_key not in cache_index: + cache_index.append(cache_metadata_key) + # Store index with same timeout as the metadata + cache.set(cache_index_key, cache_index, timeout=api.cache_timeout) + + if job: + if exclude_existing: + filtered_count = total - len(validated_devices) + job.logger.info( + f"Validation complete: {len(validated_devices)} devices passed filter, " + f"{filtered_count} filtered out (existing devices excluded)" + ) + else: + job.logger.info(f"Validation complete: {len(validated_devices)} devices ready for import") + else: + logger.info(f"Processed {len(validated_devices)} validated devices") + + if return_cache_status: + return validated_devices, from_cache + return validated_devices diff --git a/netbox_librenms_plugin/import_utils/cache.py b/netbox_librenms_plugin/import_utils/cache.py new file mode 100644 index 0000000000..140f45f79d --- /dev/null +++ b/netbox_librenms_plugin/import_utils/cache.py @@ -0,0 +1,186 @@ +"""Cache key generation and management for device import operations.""" + +import hashlib +import json +import logging + +from django.core.cache import cache + +logger = logging.getLogger(__name__) + + +def get_cache_metadata_key( + server_key: str, filters: dict, vc_enabled: bool, use_sysname: bool = True, strip_domain: bool = False +) -> str: + """ + Generate a consistent cache metadata key from filter parameters. + + Args: + server_key: LibreNMS server identifier + filters: Filter dictionary + vc_enabled: Whether VC detection is enabled + use_sysname: Whether sysName is preferred over hostname for device naming + strip_domain: Whether domain suffix is stripped from device names + + Returns: + str: Consistent cache key for metadata + """ + # Sort filter items to ensure consistent key generation; use "is not None" to preserve + # valid falsy values like 0 and False (filtering only None/missing entries). + filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v is not None) + return f"librenms_filter_cache_metadata_{server_key}_{filter_parts}_{vc_enabled}_sysname={use_sysname}_strip={strip_domain}" + + +def get_active_cached_searches(server_key: str) -> list[dict]: + """ + Retrieve all active cached searches for a server and enrich with display-friendly values. + + Enriches raw filter IDs with human-readable names by looking up location names + from cached choices and converting type codes to display names. + + Args: + server_key: LibreNMS server identifier + + Returns: + List of dicts containing cache metadata with enriched display_filters + """ + from datetime import datetime, timezone + + cache_index_key = f"librenms_cache_index_{server_key}" + cache_index = cache.get(cache_index_key, []) + + active_searches = [] + valid_cache_keys = [] + + # Get location and type choices for enriching display + location_choices = {} + type_choices = { + "": "All Types", + "network": "Network", + "server": "Server", + "storage": "Storage", + "wireless": "Wireless", + "firewall": "Firewall", + "power": "Power", + "appliance": "Appliance", + "printer": "Printer", + "loadbalancer": "Load Balancer", + "other": "Other", + } + + # Get cached location choices for enrichment; scoped by server_key so labels + # from different LibreNMS servers don't bleed into each other's filter summaries. + location_cache_key = f"librenms_locations_choices:{server_key}" + cached_locations = cache.get(location_cache_key) + if cached_locations: + location_choices = dict(cached_locations) + + for cache_key in cache_index: + metadata = cache.get(cache_key) + if metadata: + # Cache still exists, calculate time remaining + cache_timeout = metadata.get("cache_timeout", 300) + now = datetime.now(timezone.utc) + try: + cached_at_raw = metadata.get("cached_at") + cached_at = ( + datetime.fromisoformat(cached_at_raw) if cached_at_raw else datetime.fromtimestamp(0, timezone.utc) + ) + # Normalize naive datetimes (e.g., stored without tzinfo) to UTC + if cached_at.tzinfo is None: + cached_at = cached_at.replace(tzinfo=timezone.utc) + except (ValueError, TypeError): + cached_at = datetime.fromtimestamp(0, timezone.utc) + age_seconds = (now - cached_at).total_seconds() + remaining_seconds = max(0, cache_timeout - age_seconds) + + if remaining_seconds > 0: + # Add remaining time and cache key + metadata["remaining_seconds"] = int(remaining_seconds) + metadata["cache_key"] = cache_key + + # Enrich filters with human-readable display values + if "filters" in metadata: + display_filters = metadata["filters"].copy() + # Convert location ID to location name + if "location" in display_filters and display_filters["location"] in location_choices: + display_filters["location"] = location_choices[display_filters["location"]] + # Convert type code to display name + if "type" in display_filters and display_filters["type"] in type_choices: + display_filters["type"] = type_choices[display_filters["type"]] + metadata["display_filters"] = display_filters + else: + # Fallback if filters key missing + metadata["display_filters"] = {} + + active_searches.append(metadata) + valid_cache_keys.append(cache_key) + + # Clean up index if any keys have expired + if len(valid_cache_keys) < len(cache_index): + cache.set(cache_index_key, valid_cache_keys, timeout=3600) + + # Sort by most recent first + active_searches.sort(key=lambda x: x.get("cached_at", ""), reverse=True) + + return active_searches + + +def get_validated_device_cache_key( + server_key: str, + filters: dict, + device_id: int | str, + vc_enabled: bool, + use_sysname: bool = True, + strip_domain: bool = False, +) -> str: + """ + Generate a consistent cache key for validated device data. + + This ensures both synchronous and background job processing use the same + cache keys, avoiding duplicate validation work and cache entries. + + Args: + server_key: LibreNMS server key + filters: Filter dict with location, type, os, hostname, sysname, hardware keys + device_id: LibreNMS device ID + vc_enabled: Whether virtual chassis detection was enabled + use_sysname: Whether sysName is preferred over hostname for device naming + strip_domain: Whether domain suffix is stripped from device names + + Returns: + str: Cache key for the validated device + + Example: + >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) + >>> key + 'validated_device_default_e3b0c44298fc1c14_123_vc' + """ + # Sort filters for a deterministic, cross-process stable hash + filter_hash = hashlib.sha256(json.dumps(sorted(filters.items()), sort_keys=True).encode()).hexdigest()[:16] + vc_part = "vc" if vc_enabled else "novc" + return ( + f"validated_device_{server_key}_{filter_hash}_{device_id}_{vc_part}_sysname={use_sysname}_strip={strip_domain}" + ) + + +def get_import_device_cache_key(device_id: int | str, server_key: str = "default") -> str: + """ + Generate cache key for raw LibreNMS device data. + + This key is used to cache raw device data (without validation metadata) + to avoid redundant API calls when users interact with dropdowns during + the import workflow. + + Args: + device_id: LibreNMS device ID + server_key: LibreNMS server identifier for multi-server setups + + Returns: + str: Cache key for the device data + + Example: + >>> get_import_device_cache_key(123, "production") + 'import_device_data_production_123' + """ + return f"import_device_data_{server_key}_{device_id}" diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py new file mode 100644 index 0000000000..0420b87447 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -0,0 +1,917 @@ +"""Device validation, import, and fetch operations.""" + +import logging + +from dcim.models import Device, DeviceRole, DeviceType, Rack, Site +from django.core.cache import cache +from django.db import transaction +from django.db.models import Q +from django.utils import timezone +from virtualization.models import Cluster # noqa: F401 — used by test mock.patch targets + +from ..librenms_api import LibreNMSAPI +from ..utils import ( + find_matching_platform, + find_matching_site, + match_librenms_hardware_to_device_type, +) +from .cache import get_import_device_cache_key +from .virtual_chassis import ( + _generate_vc_member_name, + empty_virtual_chassis_data, + get_virtual_chassis_data, + update_vc_member_suggested_names, +) + +logger = logging.getLogger(__name__) + + +def _determine_device_name( + libre_device: dict, + use_sysname: bool = True, + strip_domain: bool = False, + device_id: int | str = None, +) -> str: + """ + Determine the device/VM name from LibreNMS data. + + Centralized logic for building device names with consistent handling of: + - sysName vs hostname preference + - Domain stripping (avoiding IP addresses) + - Fallback to device_id when name is missing + + Args: + libre_device: Device data from LibreNMS + use_sysname: If True, prefer sysName; if False, use hostname + strip_domain: If True, strip domain suffix (e.g., '.example.com') + device_id: LibreNMS device ID for fallback name generation + + Returns: + str: The determined device name + + Example: + >>> _determine_device_name({'sysName': 'router.example.com', 'hostname': 'router'}, + ... use_sysname=True, strip_domain=True) + 'router' + """ + # Determine base name based on use_sysname preference + if use_sysname: + name = libre_device.get("sysName") or libre_device.get("hostname") + else: + name = libre_device.get("hostname") or libre_device.get("sysName") + + # Fallback to device_id if no name found + if not name: + if device_id is not None: + name = f"device-{device_id}" + else: + name = libre_device.get("device_id", "unknown") + name = f"device-{name}" + + # Strip domain if requested (but not for IP addresses) + if strip_domain and name and "." in name: + try: + from ipaddress import ip_address + + ip_address(name) + # It's a valid IP address, don't strip + except ValueError: + # Not an IP, safe to strip domain + name = name.split(".")[0] + + return name + + +def validate_device_for_import( + libre_device: dict, + import_as_vm: bool = False, + api: "LibreNMSAPI" = None, + *, + include_vc_detection: bool = True, + force_vc_refresh: bool = False, + use_sysname: bool = True, + strip_domain: bool = False, + server_key: str = "default", +) -> dict: + """ + Validate if a LibreNMS device can be imported to NetBox. + + Performs comprehensive validation: + - Checks if device already exists in NetBox + - Validates required prerequisites (Site, DeviceType, DeviceRole for devices) + OR (Cluster for VMs) + - Provides smart matching for missing objects + - Detects virtual chassis/stack configuration (if API provided) + - Returns detailed validation status + + Args: + libre_device: Device data from LibreNMS + import_as_vm: If True, validate for VM import instead of device import + api: Optional LibreNMSAPI instance for virtual chassis detection + include_vc_detection: Skip VC detection when False to speed up bulk operations + force_vc_refresh: When True, bypass cached VC data and re-query LibreNMS + use_sysname: If True, prefer sysName over hostname (matches import behaviour) + strip_domain: If True, strip domain suffix from device name + + Returns: + dict: Validation result with structure: + { + 'is_ready': bool, # Can import without user intervention + 'can_import': bool, # Can import (possibly after configuration) + 'import_as_vm': bool, # Whether importing as VM + 'existing_device': Device or VirtualMachine or None, + 'issues': List[str], # Blocking issues + 'warnings': List[str], # Non-blocking warnings + 'site': { # Only for devices + 'found': bool, + 'site': Site or None, + 'match_type': str, # 'exact' or None + 'suggestions': List[Site] # Alternative suggestions + }, + 'device_type': { # Only for devices + 'found': bool, + 'device_type': DeviceType or None, + 'match_type': str, # 'exact' or None + 'suggestions': List[dict] # Device types for user selection + }, + 'device_role': { # Only for devices + 'found': bool, # Always False - requires manual selection + 'role': DeviceRole or None, + 'available_roles': List[DeviceRole] # All roles for user selection + }, + 'cluster': { # Only for VMs + 'found': bool, # Always False - requires manual selection + 'cluster': Cluster or None, + 'available_clusters': List[Cluster] # All clusters for user selection + }, + 'platform': { + 'found': bool, + 'platform': Platform or None, + 'match_type': str # 'exact' or None + } + } + + Example: + >>> validation = validate_device_for_import(libre_device) + >>> if validation['is_ready']: + ... import_single_device(libre_device['device_id']) + """ + result = { + "is_ready": False, + "can_import": False, + "import_as_vm": import_as_vm, + "resolved_name": None, # Final device name after applying user preferences + "existing_device": None, + "existing_match_type": None, # Track how existing device was matched + "serial_action": None, # None, "link", "conflict", "update_serial", "hostname_differs" + "serial_confirmed": False, # True when librenms_id match and serial matches + "serial_duplicate": False, # True when incoming serial is already on a different device + "librenms_id_needs_migration": False, # True when librenms_id is still a legacy bare int + "name_matches": False, # True when existing device name matches LibreNMS sysName + "name_sync_available": False, # True when existing device name differs from sysName + "suggested_name": None, # sysName to suggest when name_sync_available is True + "device_type_mismatch": False, # True when existing device's type differs from LibreNMS + "issues": [], + "warnings": [], + "virtual_chassis": empty_virtual_chassis_data(), + "site": { + "found": False, + "site": None, + "match_type": None, + "suggestions": [], + }, + "device_type": { + "found": False, + "device_type": None, + "match_type": None, + "suggestions": [], + }, + "device_role": { + "found": False, + "role": None, + "available_roles": [], + }, + "cluster": { + "found": False, + "cluster": None, + "available_clusters": [], + }, + "platform": {"found": False, "platform": None, "match_type": None}, + "rack": { + "found": False, + "rack": None, + "available_racks": [], + }, + "naming_criteria": None, # Populated after resolved_name is set + } + + try: + # 1. Check if device/VM already exists in NetBox + # Always check both Devices AND VMs to properly detect existing objects + librenms_id = libre_device.get("device_id") + hostname = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=strip_domain, + device_id=librenms_id, + ) + result["resolved_name"] = hostname + result["naming_criteria"] = { + "use_sysname": use_sysname, + "strip_domain": strip_domain, + "raw_sysname": libre_device.get("sysName") or "", + "raw_hostname": libre_device.get("hostname") or "", + "source": "sysname" if use_sysname and libre_device.get("sysName") else "hostname", + } + logger.debug( + f"Checking for existing device/VM: " + f"librenms_id={librenms_id} (type={type(librenms_id).__name__}), " + f"hostname={hostname}" + ) + + from virtualization.models import VirtualMachine + + # Check for existing VM first (by librenms_id custom field) + try: + from netbox_librenms_plugin.utils import find_by_librenms_id + + existing_vm = find_by_librenms_id(VirtualMachine, int(librenms_id), server_key) + except (ValueError, TypeError): + # librenms_id is not convertible to int; no match will be found + existing_vm = None + + if existing_vm: + logger.info(f"Found existing VM: {existing_vm.name} (matched by librenms_id={librenms_id})") + result["existing_device"] = existing_vm + result["existing_match_type"] = "librenms_id" + result["import_as_vm"] = True # Force VM mode since VM exists + result["can_import"] = False + + # Detect legacy bare-integer format so UI can offer a migration action. + # Direct access needed to detect legacy integer format for migration prompt: + # LibreNMSAPI.get_librenms_id() returns an int in both formats, so only the + # raw type check on custom_field_data reveals whether migration is needed. + if isinstance(existing_vm.custom_field_data.get("librenms_id"), int): + result["librenms_id_needs_migration"] = True + + # Check if name matches resolved name (accounts for use_sysname/strip_domain) + # Note: name_sync_available/suggested_name are intentionally not set for VMs + # because UpdateDeviceNameView only supports Device objects; VM name-sync + # would require a separate implementation. + if hostname and existing_vm.name == hostname: + result["name_matches"] = True + + # Check for existing Device (by librenms_id custom field) + if not result["existing_device"]: + try: + from netbox_librenms_plugin.utils import find_by_librenms_id + + existing_device = find_by_librenms_id(Device, int(librenms_id), server_key) + except (ValueError, TypeError): + # librenms_id is not convertible to int; no match will be found + existing_device = None + + if existing_device: + logger.info(f"Found existing device: {existing_device.name} (matched by librenms_id={librenms_id})") + result["existing_device"] = existing_device + result["existing_match_type"] = "librenms_id" + result["can_import"] = False + + # Detect legacy bare-integer format so UI can offer a migration action. + # Direct access needed to detect legacy integer format for migration prompt: + # LibreNMSAPI.get_librenms_id() returns an int in both formats, so only the + # raw type check on custom_field_data reveals whether migration is needed. + if isinstance(existing_device.custom_field_data.get("librenms_id"), int): + result["librenms_id_needs_migration"] = True + + # Check if name matches resolved name (VC-aware: compare against VC member name) + if hostname and existing_device.virtual_chassis and existing_device.vc_position: + vc_expected_name = _generate_vc_member_name( + hostname, + existing_device.vc_position, + serial=existing_device.serial or "", + ) + if existing_device.name == vc_expected_name: + result["name_matches"] = True + else: + result["name_sync_available"] = True + result["suggested_name"] = vc_expected_name + elif hostname and existing_device.name == hostname: + result["name_matches"] = True + elif hostname and existing_device.name != hostname: + result["name_sync_available"] = True + result["suggested_name"] = hostname + + # Check for serial drift on the linked device + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-": + if existing_device.serial and existing_device.serial == incoming_serial: + result["serial_confirmed"] = True + elif existing_device.serial and existing_device.serial != incoming_serial: + serial_conflict = ( + Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + ) + if serial_conflict: + result["serial_action"] = "conflict" + result["serial_duplicate"] = True + result["warnings"].append( + f"Serial conflict: incoming serial '{incoming_serial}' is already assigned to " + f"device '{serial_conflict.name}' (ID: {serial_conflict.pk}) in NetBox. " + f"Investigate which device should own this serial before updating." + ) + else: + result["serial_action"] = "update_serial" + result["warnings"].append( + f"Serial number differs (NetBox: '{existing_device.serial}', " + f"LibreNMS: '{incoming_serial}'). Hardware may have been replaced." + ) + + # Only check hostname/serial/IP if not already matched by librenms_id + if not result["existing_device"]: + # Check by hostname/name - Check both VMs and Devices for conflicts + existing_vm = VirtualMachine.objects.filter(name__iexact=hostname).first() + existing_device = Device.objects.filter(name__iexact=hostname).first() + + # If BOTH exist with same hostname, it's ambiguous - don't match either + if existing_vm and existing_device: + logger.warning( + f"Hostname conflict: Both VM '{existing_vm.name}' and Device " + f"'{existing_device.name}' exist with hostname '{hostname}'" + ) + result["warnings"].append( + f"Both a VM and Device exist with hostname '{hostname}' in NetBox. " + f"Cannot determine which to match. Please set the librenms_id custom field on the correct object." + ) + # Don't set existing_device, don't block import - let user proceed as new + # This allows them to import and then resolve the conflict manually + elif existing_vm: + logger.info(f"Found existing VM by hostname: {existing_vm.name}") + result["existing_device"] = existing_vm + result["existing_match_type"] = "hostname" + result["import_as_vm"] = True # Force VM mode since VM exists + result["warnings"].append( + f"VM with same hostname exists in NetBox as '{existing_vm.name}' (not linked to LibreNMS)" + ) + result["can_import"] = False + elif existing_device: + logger.info(f"Found existing device by hostname: {existing_device.name}") + result["existing_device"] = existing_device + result["existing_match_type"] = "hostname" + + # Check for serial conflict on hostname-matched device + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-" and existing_device.serial != incoming_serial: + serial_conflict = ( + Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() + ) + if serial_conflict: + result["serial_action"] = "conflict" + result["serial_duplicate"] = True + result["warnings"].append( + f"Serial conflict: incoming serial '{incoming_serial}' is already assigned to " + f"device '{serial_conflict.name}' (ID: {serial_conflict.pk}) in NetBox. " + f"Investigate which device should own this serial before importing." + ) + else: + result["serial_action"] = "update_serial" + result["warnings"].append( + f"Hostname matches but serial differs (NetBox: '{existing_device.serial}', " + f"LibreNMS: '{incoming_serial}'). Hardware may have been replaced." + ) + else: + result["warnings"].append( + f"Device with same hostname exists in NetBox as '{existing_device.name}' (not linked to LibreNMS)" + ) + + result["can_import"] = False + + # Check by serial number (strong physical match - hardware identity) + if not result["existing_device"]: + serial = libre_device.get("serial") or "" + if serial and serial != "-" and not import_as_vm: + existing_by_serial = Device.objects.filter(serial=serial).first() + if existing_by_serial: + logger.info(f"Found existing device by serial: {existing_by_serial.name} (serial={serial})") + result["existing_device"] = existing_by_serial + result["existing_match_type"] = "serial" + result["can_import"] = False + + if existing_by_serial.name and existing_by_serial.name.lower() == hostname.lower(): + result["warnings"].append( + f"Device with same serial and hostname exists as '{existing_by_serial.name}' " + f"(not linked to LibreNMS)" + ) + result["serial_action"] = "link" + else: + result["warnings"].append( + f"Device with same serial ({serial}) exists as '{existing_by_serial.name}' " + f"but hostname differs (LibreNMS: '{hostname}'). Device may have been reinstalled." + ) + result["serial_action"] = "hostname_differs" + + # Check by primary IP (weaker match, IP could be reassigned) - only for devices + if not result["existing_device"]: + primary_ip = libre_device.get("ip") + if primary_ip and not import_as_vm: + from ipam.models import IPAddress + + existing_ip = IPAddress.objects.filter(address__net_host=primary_ip).first() + if existing_ip and existing_ip.assigned_object: + device = ( + existing_ip.assigned_object.device + if hasattr(existing_ip.assigned_object, "device") + else None + ) + if device: + result["existing_device"] = device + result["existing_match_type"] = "primary_ip" + result["warnings"].append( + f"IP address {primary_ip} already assigned to device '{device.name}' (not linked to LibreNMS)" + ) + result["can_import"] = False + + # Refresh local variable to reflect any VM-mode adjustments made during detection + # (e.g. existing VM found by hostname sets result["import_as_vm"] = True) + import_as_vm = result["import_as_vm"] + + # Validate based on import type (Device or VM) + if import_as_vm: + # 2. For VMs: Validate Cluster (required) - Must be manually selected + result["cluster"]["found"] = False + result["issues"].append("Cluster must be manually selected before importing as VM") + # Provide list of available clusters for user selection (cached) + cache_key = "librenms_import_all_clusters" + all_clusters = cache.get(cache_key) + if all_clusters is None: + all_clusters = list(Cluster.objects.all()) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, all_clusters, cache_timeout) + result["cluster"]["available_clusters"] = all_clusters + + # Skip device-specific validations for VMs + result["site"]["found"] = True # Not required for VMs + result["device_type"]["found"] = True # Not required for VMs + result["device_role"]["found"] = True # Not required for VMs + + else: + # 2. For Devices: Validate Site (required) + location = libre_device.get("location", "") + site_match = find_matching_site(location) + result["site"] = site_match + + if not site_match["found"]: + result["issues"].append(f"No matching site found for location: '{location}'") + # Get alternative suggestions + if location: + all_sites = Site.objects.all()[:10] # Limit for performance + result["site"]["suggestions"] = list(all_sites) + + # 3. Validate DeviceType (required) + hardware = libre_device.get("hardware", "") + dt_match = match_librenms_hardware_to_device_type(hardware) + # Update result keys individually to preserve the existing schema (especially "found") + result["device_type"]["found"] = dt_match["matched"] + result["device_type"]["device_type"] = dt_match.get("device_type") + result["device_type"]["match_type"] = dt_match.get("match_type") + + if not dt_match["matched"]: + result["issues"].append(f"No matching device type found for hardware: '{hardware}'") + # Get some device types for user to choose from + all_device_types = DeviceType.objects.all()[:10] + result["device_type"]["suggestions"] = [ + { + "device_type": dt, + "similarity": 0.0, # No fuzzy matching, just showing options + "match_field": None, + } + for dt in all_device_types + ] + + # 4. DeviceRole (required) - Must be manually selected by user + logger.debug(f"[{hostname}] Issues BEFORE adding role issue: {result['issues']}") + result["device_role"]["found"] = False + result["issues"].append("Device role must be manually selected before import") + logger.debug(f"[{hostname}] Issues AFTER adding role issue: {result['issues']}") + # Provide list of available roles for user selection (cached) + cache_key = "librenms_import_all_roles" + all_roles = cache.get(cache_key) + if all_roles is None: + all_roles = list(DeviceRole.objects.all()) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, all_roles, cache_timeout) + result["device_role"]["available_roles"] = all_roles + + # 4b. Rack (optional) - Provide available racks for the matched site + if site_match["found"] and site_match["site"]: + site = site_match["site"] + # Use cache to optimize rack lookups per site + cache_key = f"librenms_import_racks_site_{site.pk}" + available_racks = cache.get(cache_key) + + if available_racks is None: + # Query racks for this site - include both: + # 1. Racks assigned to locations within the site + # 2. Racks directly assigned to the site (without location) + available_racks = list( + Rack.objects.filter(Q(location__site=site) | Q(site=site)) + .select_related("location", "site") + .order_by("location__name", "name") + ) + # Use API cache timeout if available, otherwise use default 5 minutes + cache_timeout = api.cache_timeout if api else 300 + cache.set(cache_key, available_racks, cache_timeout) + + result["rack"]["available_racks"] = available_racks + # Rack is optional, don't add to issues + result["rack"]["found"] = True # Mark as "found" even if None (optional field) + + # Skip VM-specific validations for devices + result["cluster"]["found"] = True # Not required for devices + + # 5. Match Platform (optional - same for both devices and VMs) + os = libre_device.get("os", "") + platform_match = find_matching_platform(os) + result["platform"] = platform_match + + if not platform_match["found"] and os: + result["warnings"].append(f"No matching platform found for OS: '{os}'") + + # 6. Additional validations + if not hostname: + result["issues"].append("Device has no hostname") + + # 7. Virtual chassis detection (only for devices, not VMs) + if include_vc_detection and not import_as_vm and api is not None: + device_id = libre_device.get("device_id") + if device_id: + try: + logger.debug(f"Calling get_virtual_chassis_data for device {device_id}") + vc_detection = get_virtual_chassis_data(api, device_id, force_refresh=force_vc_refresh) + logger.debug( + f"VC detection result: is_stack={vc_detection.get('is_stack')}, " + f"member_count={vc_detection.get('member_count')}, " + f"members={len(vc_detection.get('members', []))}" + ) + if vc_detection: + result["virtual_chassis"] = vc_detection + if vc_detection["is_stack"]: + logger.debug( + f"Virtual chassis CONFIRMED for device {hostname}: " + f"{vc_detection['member_count']} members" + ) + result["virtual_chassis"] = update_vc_member_suggested_names(vc_detection, hostname) + except Exception as e: + logger.exception(f"Exception during VC detection for device {hostname}: {e}") + result["virtual_chassis"]["detection_error"] = str(e) + else: + logger.debug(f"No device_id found for {hostname}") + + # 8. Determine if device/VM is ready to import + if result["existing_device"]: + # Already matched - can_import was already set to False + result["is_ready"] = False + # Populate role from existing device so the modal shows it + existing = result["existing_device"] + if hasattr(existing, "role") and existing.role: + result["device_role"]["found"] = True + result["device_role"]["role"] = existing.role + + # Check for device type mismatch between existing device and LibreNMS + if hasattr(existing, "device_type") and existing.device_type: + librenms_dt = result["device_type"].get("device_type") + if librenms_dt and existing.device_type.pk != librenms_dt.pk: + result["device_type_mismatch"] = True + result["warnings"].append( + f"Device type mismatch: NetBox has '{existing.device_type}' " + f"but LibreNMS reports '{librenms_dt}'. " + f"This may indicate the wrong device was matched." + ) + else: + result["can_import"] = len(result["issues"]) == 0 + + if import_as_vm: + # For VMs: only cluster is required + result["is_ready"] = result["can_import"] and result["cluster"]["found"] + else: + # For Devices: site, device_type, and device_role are required + result["is_ready"] = ( + result["can_import"] + and result["site"]["found"] + and result["device_type"]["found"] + and result["device_role"]["found"] + ) + + logger.debug( + f"Validation for {libre_device.get('hostname')} ({'VM' if import_as_vm else 'Device'}): " + f"issues={len(result['issues'])}, can_import={result['can_import']}, " + f"issues_list={result['issues']}" + ) + + return result + + except Exception as e: + logger.exception(f"Error validating device for import: {libre_device.get('hostname', 'unknown')}") + result["issues"].append(f"Validation error: {str(e)}") + return result + + +def import_single_device( + device_id: int, + server_key: str = None, + validation: dict = None, + manual_mappings: dict = None, + sync_options: dict = None, + libre_device: dict = None, +) -> dict: + """ + Import a single LibreNMS device to NetBox. + + Args: + device_id: LibreNMS device ID + server_key: LibreNMS server configuration key + validation: Pre-computed validation dict (optional) + manual_mappings: Manual object mappings (optional): + - site_id: NetBox Site ID + - device_type_id: NetBox DeviceType ID + - device_role_id: NetBox DeviceRole ID + - platform_id: NetBox Platform ID (optional) + - rack_id: NetBox Rack ID (optional) + sync_options: Sync options (optional): + - sync_interfaces: bool (default True) + - sync_cables: bool (default True) + - sync_ips: bool (default True) + - sync_fields: bool (default True) + libre_device: Pre-fetched LibreNMS device data (optional). + If provided, skips API call to fetch device info. + + Returns: + dict: Import result with structure: + { + 'success': bool, + 'device': Device object or None, + 'message': str, + 'error': str or None, + 'synced': { + 'interfaces': int, + 'cables': int, + 'ip_addresses': int + } + } + """ + try: + api = LibreNMSAPI(server_key=server_key) + + # Use pre-fetched device data if provided, otherwise fetch from API + if libre_device is None: + success, libre_device = api.get_device_info(device_id) + if not success or not libre_device: + return { + "success": False, + "device": None, + "message": "", + "error": f"Failed to retrieve device {device_id} from LibreNMS", + "synced": {}, + } + + # Validate device if validation not provided + if validation is None: + use_sysname_opt = sync_options.get("use_sysname", True) if sync_options else True + strip_domain_opt = sync_options.get("strip_domain", False) if sync_options else False + validation = validate_device_for_import( + libre_device, + use_sysname=use_sysname_opt, + strip_domain=strip_domain_opt, + server_key=api.server_key, + ) + + # Check if device already exists + if validation.get("existing_device"): + return { + "success": False, + "device": validation["existing_device"], + "message": "", + "error": f"Device already exists: {validation['existing_device'].name}", + "synced": {}, + } + + # Use validation-derived matches, allow manual mappings to override specific fields + site = validation["site"].get("site") + device_type = validation["device_type"].get("device_type") + device_role = validation["device_role"].get("role") + platform = validation["platform"].get("platform") + rack = validation.get("rack", {}).get("rack") + + if manual_mappings: + site = Site.objects.filter(id=manual_mappings.get("site_id")).first() or site + device_type = DeviceType.objects.filter(id=manual_mappings.get("device_type_id")).first() or device_type + device_role = DeviceRole.objects.filter(id=manual_mappings.get("device_role_id")).first() or device_role + + platform_id = manual_mappings.get("platform_id") + if platform_id: + from dcim.models import Platform + + platform = Platform.objects.filter(id=platform_id).first() or platform + + rack_id = manual_mappings.get("rack_id") + if rack_id: + rack = Rack.objects.select_related("location", "site").filter(id=rack_id).first() or rack + + rack = rack or validation.get("rack", {}).get("rack") + + # Validate required fields + if not site: + return { + "success": False, + "device": None, + "message": "", + "error": "Site is required but not provided", + "synced": {}, + } + if not device_type: + return { + "success": False, + "device": None, + "message": "", + "error": "Device type is required but not provided", + "synced": {}, + } + if not device_role: + return { + "success": False, + "device": None, + "message": "", + "error": "Device role is required but not provided", + "synced": {}, + } + + # Create device in NetBox + with transaction.atomic(): + # Determine device name based on sync options + use_sysname = sync_options.get("use_sysname", True) if sync_options else True + strip_domain = sync_options.get("strip_domain", False) if sync_options else False + + device_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=strip_domain, + device_id=device_id, + ) + + # Generate import timestamp comment + import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") + + device_data = { + "name": device_name, + "site": site, + "device_type": device_type, + "role": device_role, + "status": "active" if libre_device.get("status") == 1 else "offline", + "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", + "custom_field_data": {"librenms_id": {api.server_key: int(device_id)}}, + } + + # Add optional fields + if platform: + device_data["platform"] = platform + + if rack: + device_data["rack"] = rack + + serial = libre_device.get("serial", "") + if serial and serial != "-": + device_data["serial"] = serial + + location_name = libre_device.get("location", "") + if location_name and location_name != "-": + from dcim.models import Location + + # Try to find matching location within the site + location = Location.objects.filter(site=site, name__iexact=location_name).first() + if location: + device_data["location"] = location + + # Create the device + device = Device(**device_data) + device.full_clean() + device.save() + + # Sync additional data based on options + sync_options = sync_options or {} + synced = {"interfaces": 0, "cables": 0, "ip_addresses": 0} + + try: + # Sync interfaces + if sync_options.get("sync_interfaces", True): + # This is simplified - would need proper request context + # For now, just log that it should be done + logger.info(f"Interface sync should be performed for device {device.name}") + + # Sync cables + if sync_options.get("sync_cables", True): + logger.info(f"Cable sync should be performed for device {device.name}") + + # Sync IP addresses + if sync_options.get("sync_ips", True): + logger.info(f"IP address sync should be performed for device {device.name}") + + except Exception as e: + logger.warning(f"Error during post-import sync: {str(e)}") + # Don't fail the import if sync fails + + return { + "success": True, + "device": device, + "message": f"Successfully imported device: {device.name}", + "error": None, + "synced": synced, + } + + except Exception as e: + logger.exception(f"Error importing device {device_id}") + return { + "success": False, + "device": None, + "message": "", + "error": str(e), + "synced": {}, + } + + +def get_librenms_device_by_id(api: LibreNMSAPI, device_id: int) -> dict: + """ + Retrieve a single device from LibreNMS by ID. + + Args: + api: LibreNMSAPI instance + device_id: LibreNMS device ID + + Returns: + Device dictionary or None if not found + """ + try: + # Use the dedicated API endpoint to get device by ID + success, device = api.get_device_info(device_id) + if success and device: + return device + + logger.warning(f"Device {device_id} not found in LibreNMS") + return None + except Exception as e: + logger.exception(f"Failed to get device {device_id} from LibreNMS: {e}") + return None + + +def fetch_device_with_cache( + device_id: int, + api: LibreNMSAPI, + server_key: str = None, + libre_devices_cache: dict = None, +) -> dict | None: + """ + Fetch LibreNMS device from cache or API with automatic caching. + + Checks three sources in order: + 1. Pre-fetched cache dict (if provided) + 2. Django cache (Redis/memory) + 3. LibreNMS API (caches result for future use) + + This function consolidates the device fetching pattern used throughout + the import workflow, eliminating code duplication. + + Args: + device_id: LibreNMS device ID to fetch + api: LibreNMSAPI instance for fallback API calls + server_key: Optional server key for multi-server setups (defaults to api.server_key) + libre_devices_cache: Optional pre-fetched device cache dict + + Returns: + Device dict from LibreNMS, or None if not found + + Example: + >>> # Simple usage + >>> libre_device = fetch_device_with_cache(123, api) + >>> if libre_device: + ... print(libre_device['hostname']) + >>> + >>> # With pre-fetched cache dict + >>> cache_dict = {123: {...}, 456: {...}} + >>> libre_device = fetch_device_with_cache(123, api, libre_devices_cache=cache_dict) + """ + # Check pre-fetched cache dict first (fastest) + if libre_devices_cache and device_id in libre_devices_cache: + return libre_devices_cache[device_id] + + # Check Django cache + cache_key = get_import_device_cache_key(device_id, server_key or api.server_key) + libre_device = cache.get(cache_key) + + if not libre_device: + # Fallback to API fetch + libre_device = get_librenms_device_by_id(api, device_id) + if libre_device: + # Cache for future use + cache.set(cache_key, libre_device, timeout=api.cache_timeout) + + return libre_device diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py new file mode 100644 index 0000000000..3e12658066 --- /dev/null +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -0,0 +1,268 @@ +"""Device filtering and retrieval from LibreNMS.""" + +import hashlib +import json +import logging +from typing import List + +from django.core.cache import cache + +from ..librenms_api import LibreNMSAPI + +logger = logging.getLogger(__name__) + + +def get_device_count_for_filters( + api: LibreNMSAPI, + filters: dict, + clear_cache: bool = False, + show_disabled: bool = True, +) -> int: + """ + Get count of LibreNMS devices matching filters. + + This is a lightweight function to determine device count for background job + decision making. Uses the same caching as get_librenms_devices_for_import(). + + Args: + api: LibreNMS API client instance + filters: Filter dict with location, type, os, hostname, sysname keys + clear_cache: Whether to force cache refresh + show_disabled: Whether to include disabled devices + + Returns: + int: Count of devices matching filters + """ + devices = get_librenms_devices_for_import(api, filters=filters, force_refresh=clear_cache) + + # Filter out disabled devices if requested. LibreNMS's "disabled" field (1=disabled, + # 0=enabled) reflects manual device disablement; "status" reflects SNMP reachability. + # show_disabled controls the former: hidden when disabled==1, shown regardless of status. + if not show_disabled: + devices = [d for d in devices if int(d.get("disabled", 0)) != 1] + + return len(devices) + + +def get_librenms_devices_for_import( + api: LibreNMSAPI = None, + filters: dict = None, + server_key: str = None, + *, + force_refresh: bool = False, + return_cache_status: bool = False, +) -> List[dict] | tuple[List[dict], bool]: + """ + Retrieve LibreNMS devices based on filters. + + Args: + api: LibreNMSAPI instance (if not provided, creates one with server_key) + filters: Dict containing filter parameters: + - location: LibreNMS location/site filter + - type: Device type filter + - os: Operating system filter + - hostname: Hostname filter (partial match) + - sysname: System name filter (partial match) + - status: Device status filter (1=up, 0=down) + - disabled: Include disabled devices (0=active only, 1=all) + server_key: Key for specific server configuration (used if api not provided) + force_refresh: When True, bypass the cache and fetch fresh data + return_cache_status: When True, returns (devices, from_cache) tuple + + Returns: + List of device dictionaries from LibreNMS, or tuple of (devices, from_cache) + if return_cache_status is True. from_cache=True means data was loaded from + existing cache; from_cache=False means data was just fetched from LibreNMS. + """ + try: + # Use provided API instance or create a new one + if api is None: + api = LibreNMSAPI(server_key=server_key) + + # Build LibreNMS API filters using the type/query format + # LibreNMS API v0 expects ?type=X&query=Y format, not direct parameters + # NOTE: API only supports ONE type/query pair, so we'll use the most + # specific filter for the API and apply others client-side + api_filters = {} + client_filters = {} # Filters to apply after fetching from API + + if filters: + # Check for status filter first - it has special handling + if filters.get("status") is not None: + # Normalize to int: form fields send strings ("1"/"0"), API may send ints + try: + status_val = int(filters["status"]) + except (ValueError, TypeError): + status_val = None + # Status filter uses special types that don't need query param + if status_val == 1: + api_filters["type"] = "up" + elif status_val == 0: + api_filters["type"] = "down" + + # Save ALL other filters for client-side filtering when status is used + if filters.get("location"): + client_filters["location"] = filters["location"] + if filters.get("type"): + client_filters["type"] = filters["type"] + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + else: + # Priority order for type/query filters: location > type > os > hostname > sysname + # Note: When sysname is combined with other filters, it's applied client-side for partial matching + # When sysname is alone, it uses API exact match (type=sysName) + # Note: hardware is always applied client-side for partial matching + # Use first available for API, save others for client-side filtering + if filters.get("location"): + api_filters["type"] = "location_id" + api_filters["query"] = filters["location"] + # Save remaining filters for client-side + if filters.get("type"): + client_filters["type"] = filters["type"] + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("type"): + api_filters["type"] = "type" + api_filters["query"] = filters["type"] + # Save remaining filters for client-side + if filters.get("os"): + client_filters["os"] = filters["os"] + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("os"): + api_filters["type"] = "os" + api_filters["query"] = filters["os"] + # Save remaining filters for client-side + if filters.get("hostname"): + client_filters["hostname"] = filters["hostname"] + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("hostname"): + api_filters["type"] = "hostname" + api_filters["query"] = filters["hostname"] + # Save sysname and hardware for client-side + if filters.get("sysname"): + client_filters["sysname"] = filters["sysname"] + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("sysname"): + # sysname-only filter: Use API exact match (type=sysName&query=) + # This is safe - returns empty if no exact match found + api_filters["type"] = "sysName" + api_filters["query"] = filters["sysname"] + # Save hardware for client-side + if filters.get("hardware"): + client_filters["hardware"] = filters["hardware"] + elif filters.get("hardware"): + # hardware-only filter: apply client-side for partial matching + client_filters["hardware"] = filters["hardware"] + + # Note: disabled filter isn't directly supported by LibreNMS API + # We'll filter client-side if needed + + # Use caching to avoid repeated API calls + # Include both API and client filters in cache key (deterministic, cross-process stable). + # Use api.server_key (always resolved) rather than the raw server_key arg (may differ). + def _hash(d): + return hashlib.sha256( + json.dumps(sorted(d.items()) if isinstance(d, dict) else d, sort_keys=True).encode() + ).hexdigest()[:16] + + cache_key = f"librenms_devices_import_{api.server_key}_{_hash(api_filters)}_{_hash(client_filters)}" + from_cache = False + + if force_refresh: + cache.delete(cache_key) + else: + cached_result = cache.get(cache_key) + if cached_result is not None: + # No need to deepcopy - cached data isn't mutated + devices = cached_result + from_cache = True + if return_cache_status: + return devices, from_cache + return devices + + success, devices = api.list_devices(api_filters if api_filters else None) + + if not success: + logger.error(f"Failed to retrieve devices from LibreNMS: {devices}") + if return_cache_status: + return [], False + return [] + + # Apply client-side filters if any + if client_filters: + devices = _apply_client_filters(devices, client_filters) + + # Cache using configured timeout (default 300s) + # No need to deepcopy - Django's cache backend handles serialization + cache.set(cache_key, devices, timeout=api.cache_timeout) + + if return_cache_status: + return devices, from_cache + return devices + + except Exception: + logger.exception("Error retrieving LibreNMS devices for import") + if return_cache_status: + return [], False + return [] + + +def _apply_client_filters(devices: List[dict], filters: dict) -> List[dict]: + """ + Apply client-side filters to device list. + + Args: + devices: List of device dicts from LibreNMS + filters: Dict of filters to apply (location, type, os, hostname, sysname) + + Returns: + Filtered list of devices + """ + filtered = devices + + if filters.get("location"): + location_id = str(filters["location"]) + filtered = [d for d in filtered if str(d.get("location_id", "")) == location_id] + + if filters.get("type"): + device_type = filters["type"].lower() + filtered = [d for d in filtered if d.get("type", "").lower() == device_type] + + if filters.get("os"): + os_filter = filters["os"].lower() + filtered = [d for d in filtered if os_filter in d.get("os", "").lower()] + + if filters.get("hostname"): + hostname_filter = filters["hostname"].lower() + filtered = [d for d in filtered if hostname_filter in d.get("hostname", "").lower()] + + if filters.get("sysname"): + sysname_filter = filters["sysname"].lower() + filtered = [d for d in filtered if sysname_filter in d.get("sysName", "").lower()] + + if filters.get("hardware"): + hardware_filter = filters["hardware"].lower() + filtered = [d for d in filtered if hardware_filter in (d.get("hardware") or "").lower()] + + return filtered diff --git a/netbox_librenms_plugin/import_utils/permissions.py b/netbox_librenms_plugin/import_utils/permissions.py new file mode 100644 index 0000000000..742e50893c --- /dev/null +++ b/netbox_librenms_plugin/import_utils/permissions.py @@ -0,0 +1,44 @@ +"""Permission check helpers for device import operations.""" + +from django.core.exceptions import PermissionDenied + + +def check_user_permissions(user, permissions): + """ + Check if user has all required permissions. + + Args: + user: The user object to check permissions for + permissions: List of permission strings (e.g., ['dcim.add_device', 'dcim.add_interface']) + + Returns: + tuple: (has_all_permissions: bool, missing_permissions: list[str]) + + Raises: + PermissionDenied: If user is None (no user context available) + """ + if user is None: + raise PermissionDenied("No user context available for permission check") + + missing = [perm for perm in permissions if not user.has_perm(perm)] + return (len(missing) == 0, missing) + + +def require_permissions(user, permissions, action_description="perform this action"): + """ + Require user has all permissions, raising PermissionDenied if not. + + Args: + user: The user object to check permissions for + permissions: List of permission strings + action_description: Human-readable description for error message + + Raises: + PermissionDenied: If user lacks any required permission + """ + has_perms, missing = check_user_permissions(user, permissions) + if not has_perms: + missing_str = ", ".join(missing) + raise PermissionDenied( + f"You do not have permission to {action_description}. Missing permissions: {missing_str}" + ) diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py new file mode 100644 index 0000000000..b95f32eaec --- /dev/null +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -0,0 +1,470 @@ +"""Virtual chassis detection, creation, and management.""" + +import logging +from typing import List + +from dcim.models import Device, VirtualChassis +from django.core.cache import cache +from django.db import transaction + +from ..librenms_api import LibreNMSAPI + +logger = logging.getLogger(__name__) + + +def empty_virtual_chassis_data() -> dict: + """Public helper for callers that need a blank VC payload.""" + + return { + "is_stack": False, + "member_count": 0, + "members": [], + "detection_error": None, + } + + +def _clone_virtual_chassis_data(data: dict | None) -> dict: + """Return a defensive copy of cached VC data to avoid shared references.""" + + if not data: + return empty_virtual_chassis_data() + + members = [] + for idx, member in enumerate(data.get("members", [])): + member_copy = member.copy() + raw_position = member_copy.get("position", idx + 1) + try: + member_copy["position"] = int(raw_position) + except (TypeError, ValueError): + member_copy["position"] = idx + 1 # 1-based fallback; position 0 is invalid + members.append(member_copy) + + member_count = data.get("member_count") or len(members) + + return { + "is_stack": bool(data.get("is_stack")), + "member_count": member_count, + "members": members, + "detection_error": data.get("detection_error"), + } + + +_VC_CACHE_VERSION = "v1" + + +def _vc_cache_key(api: LibreNMSAPI, device_id: int | str) -> str: + server_key = getattr(api, "server_key", "default") + return f"librenms_vc_detection_{_VC_CACHE_VERSION}_{server_key}_{device_id}" + + +def get_virtual_chassis_data(api: LibreNMSAPI, device_id: int | str, *, force_refresh: bool = False) -> dict: + """Fetch (and cache) virtual chassis data for a LibreNMS device.""" + + if not api or device_id is None: + return empty_virtual_chassis_data() + + cache_key = _vc_cache_key(api, device_id) + if not force_refresh: + cached = cache.get(cache_key) + if cached is not None: + return _clone_virtual_chassis_data(cached) + + detection_data = detect_virtual_chassis_from_inventory(api, device_id) + if detection_data and "detection_error" not in detection_data: + detection_data["detection_error"] = None + + cache_value = _clone_virtual_chassis_data(detection_data) if detection_data else empty_virtual_chassis_data() + + cache_timeout = getattr(api, "cache_timeout", 300) or 300 + cache.set(cache_key, cache_value, timeout=cache_timeout) + return _clone_virtual_chassis_data(cache_value) + + +def prefetch_vc_data_for_devices(api: LibreNMSAPI, device_ids: List[int], *, force_refresh: bool = False) -> None: + """ + Pre-warm the virtual chassis cache for multiple devices. + + This eliminates the 0.5-1s delay when rendering the import table + by proactively fetching VC data before validation. + + Args: + api: LibreNMSAPI instance + device_ids: List of LibreNMS device IDs to prefetch VC data for + force_refresh: When True, bypass cache and fetch fresh data + + Example: + >>> # Before rendering import table + >>> prefetch_vc_data_for_devices(api, [123, 124, 125]) + >>> # Now all validate_device_for_import() calls hit cache instantly + """ + if not api or not device_ids: + return + + logger.debug(f"Pre-warming VC cache for {len(device_ids)} devices") + + for idx, device_id in enumerate(device_ids): + # This populates the cache if empty, or skips if already cached + try: + get_virtual_chassis_data(api, device_id, force_refresh=force_refresh) + except (BrokenPipeError, ConnectionError, IOError, OSError) as e: + logger.warning(f"Connection error during VC prefetch at device {idx}: {e}") + # Stop processing if connection is broken + return + except Exception as e: + # Log but continue for other errors + logger.warning(f"Error prefetching VC data for device {device_id}: {e}") + + logger.debug(f"VC cache warming complete for {len(device_ids)} devices") + + +def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> dict: + """ + Detect if device is a stack/Virtual Chassis by analyzing ENTITY-MIB inventory. + Vendor-agnostic using standard hierarchical structure. + + Args: + api: LibreNMSAPI instance + device_id: LibreNMS device ID + + Returns: + dict with structure: + { + 'is_stack': bool, + 'member_count': int, + 'members': [ + { + 'serial': str, + 'position': int, + 'model': str, + 'name': str, + 'index': int, + 'description': str, + 'suggested_name': str # Generated using master device name + } + ] + } + Returns None if not a stack or detection fails. + + Detection Logic: + 1. Check root level (entPhysicalContainedIn=0) for parent container + 2. Find parent index (entPhysicalClass='stack' or 'chassis') + 3. Get children chassis at that parent's index + 4. If multiple chassis found -> Stack detected + """ + try: + # Get the master device info to use for naming + success, device_info = api.get_device_info(device_id) + master_name = None + if success and device_info: + master_name = device_info.get("sysName") or device_info.get("hostname") + + # Step 1: Get root level items + success, root_items = api.get_inventory_filtered(device_id, ent_physical_contained_in=0) + + if not success or not root_items: + logger.debug(f"No root inventory items found for device {device_id}") + return None + + # Step 2: Find parent container index + # Could be class="stack" or the main "chassis" + parent_index = None + for item in root_items: + item_class = item.get("entPhysicalClass") + if item_class in ["stack", "chassis"]: + parent_index = item.get("entPhysicalIndex") + logger.debug(f"VC detection: Found parent container at index {parent_index} for device {device_id}") + break + + if parent_index is None: + return None + + # Step 3: Get children chassis at next level + success, child_items = api.get_inventory_filtered( + device_id, + ent_physical_class="chassis", + ent_physical_contained_in=parent_index, + ) + + if not success: + return None + + # Filter for chassis only (in case API filter didn't work) + chassis_items = [item for item in (child_items or []) if item.get("entPhysicalClass") == "chassis"] + + # Step 4: Multiple chassis = stack + if len(chassis_items) <= 1: + return None + + # Step 5: Extract member info + members = [] + for idx, chassis in enumerate(chassis_items): + # entPhysicalParentRelPos is 1-based; fall back to idx+1 (not idx) so + # position 0 is never produced — VC positions must be ≥ 1. + raw_position = chassis.get("entPhysicalParentRelPos", idx + 1) + try: + position = int(raw_position) + except (TypeError, ValueError): + position = idx + 1 + member_data = { + "serial": chassis.get("entPhysicalSerialNum", ""), + "position": position, + "model": chassis.get("entPhysicalModelName", ""), + "name": chassis.get("entPhysicalName", ""), + "index": chassis.get("entPhysicalIndex"), + "description": chassis.get("entPhysicalDescr", ""), + } + + # Generate suggested name if we have master name. + # position is already 1-based, so pass it directly (no +1). + if master_name: + member_data["suggested_name"] = _generate_vc_member_name(master_name, position) + else: + member_data["suggested_name"] = f"Member-{position}" + + members.append(member_data) + + # Sort by position + members.sort(key=lambda m: m["position"]) + + logger.info(f"Detected stack with {len(members)} members for device {device_id}") + + return {"is_stack": True, "member_count": len(members), "members": members} + + except Exception as e: + logger.exception(f"Error detecting virtual chassis for device {device_id}: {e}") + return None + + +def _load_vc_member_name_pattern() -> str: + """Load the VC member name pattern from settings, with fallback to default.""" + from ..models import LibreNMSSettings + + try: + settings = LibreNMSSettings.objects.first() + return settings.vc_member_name_pattern if settings else "-M{position}" + except Exception as e: + logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") + return "-M{position}" + + +def _generate_vc_member_name(master_name: str, position: int, serial: str = None, pattern: str = None) -> str: + """ + Generate name for VC member device using configured pattern from settings. + + Args: + master_name: Name of the master/primary device + position: VC position number + serial: Optional serial number of the member device + pattern: Optional pre-loaded name pattern; if None, loaded from settings. + Pass a pre-loaded pattern when calling inside a loop to avoid + repeated DB queries. + + Returns: + Generated member device name + + Examples: + pattern="-M{position}" -> "switch01-M2" + pattern=" ({position})" -> "switch01 (2)" + pattern="-SW{position}" -> "switch01-SW2" + pattern=" [{serial}]" -> "switch01 [ABC123]" + """ + if pattern is None: + pattern = _load_vc_member_name_pattern() + + # Prepare format variables + format_vars = { + "master_name": master_name, + "position": position, + "serial": serial or "", + } + + # Apply pattern - pattern should be suffix/prefix, not full name + try: + formatted_suffix = pattern.format(**format_vars) + return f"{master_name}{formatted_suffix}" + except KeyError as e: + logger.error(f"Invalid placeholder in VC naming pattern '{pattern}': {e}. Using default.") + return f"{master_name}-M{position}" + + +def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: + """ + Regenerate suggested VC member names using the actual master device name. + + This ensures preview shows accurate names after use_sysname and strip_domain + are applied to the master device name. + + Args: + vc_data: Virtual chassis detection data dict + master_name: The actual name that will be used for master device in NetBox + + Returns: + Updated vc_data dict with corrected suggested_name for each member + """ + if not vc_data or not vc_data.get("is_stack"): + return vc_data + + # Load naming pattern once to avoid a DB query per member + vc_pattern = _load_vc_member_name_pattern() + for idx, member in enumerate(vc_data.get("members", [])): + raw_position = member.get("position", idx) + try: + base_position = int(raw_position) + except (TypeError, ValueError): + base_position = idx + position = base_position + 1 # Convert to 1-based position + member["position"] = base_position + member["suggested_name"] = _generate_vc_member_name( + master_name, position, serial=member.get("serial"), pattern=vc_pattern + ) + + return vc_data + + +def create_virtual_chassis_with_members(master_device: Device, members_info: list, libre_device: dict): + """ + Create Virtual Chassis and member devices from detection info. + + This function creates a NetBox VirtualChassis with the master device + and all detected member devices, wrapped in a transaction for safety. + + Args: + master_device: The imported device (becomes VC master) + members_info: List of member dicts from VC detection + libre_device: Original LibreNMS device data + + Returns: + VirtualChassis: The created virtual chassis instance + + Raises: + ValidationError: If member count validation fails + IntegrityError: If duplicate serials/names are detected + Exception: For other creation errors + + Example members_info: + [ + {'serial': 'ABC123', 'position': 0, 'model': 'C9300-48U', 'name': 'Switch 1'}, + {'serial': 'ABC124', 'position': 1, 'model': 'C9300-48U', 'name': 'Switch 2'} + ] + """ + + # original_master_name is still referenced in warning messages inside the atomic block. + original_master_name = master_device.name + + try: + with transaction.atomic(): + # Rename master device to include position 1 pattern + master_device_new_name = _generate_vc_member_name(original_master_name, 1, serial=master_device.serial) + + # Check if renamed master conflicts with existing device + if Device.objects.filter(name=master_device_new_name).exclude(pk=master_device.pk).exists(): + logger.warning( + f"Cannot rename master to '{master_device_new_name}' - name already exists. " + f"Keeping original name '{original_master_name}'" + ) + master_base_name = original_master_name + else: + master_device.name = master_device_new_name + master_base_name = original_master_name + + # Create VC using original base name + vc_name = master_base_name + vc = VirtualChassis.objects.create( + name=vc_name, + master=master_device, + domain=f"librenms-{libre_device.get('device_id', master_device.pk)}", + ) + + # Update master device + master_device.virtual_chassis = vc + master_device.vc_position = 1 # Master is position 1 + master_device.save() + + # Create member devices for remaining positions + position = 2 # Start at 2 (master is 1) + members_created = 0 + # Load naming pattern once to avoid a DB query per member + vc_pattern = _load_vc_member_name_pattern() + + for member in members_info: + # Skip if this is the master's serial (only when both serials are non-empty) + if member.get("serial") and member.get("serial") == master_device.serial: + continue + + serial = member.get("serial") + + member_rack = master_device.rack + member_location = master_device.location or ( + member_rack.location if member_rack and member_rack.location else None + ) + + # Check for duplicate serial + if serial and Device.objects.filter(serial=serial).exists(): + logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") + continue + + # Prefer the discovered SNMP position; fall back to sequential counter. + # Normalize discovered_pos: 0 is not a valid VC position, treat as absent. + try: + discovered_pos = int(member.get("position")) if member.get("position") is not None else None + except (TypeError, ValueError): + discovered_pos = None + if discovered_pos is not None and discovered_pos < 1: + discovered_pos = None # 0 is invalid for vc_position; fall back to counter + chosen_pos = discovered_pos if discovered_pos is not None else position + # Advance the sequential counter: + # - if discovered_pos was used, advance counter past it to avoid future reuse; + # - if counter was consumed as fallback, increment it normally. + if discovered_pos is None: + position += 1 + else: + position = max(position, discovered_pos + 1) + + member_name = _generate_vc_member_name(master_base_name, chosen_pos, serial=serial, pattern=vc_pattern) + + # Check for duplicate name + if Device.objects.filter(name=member_name).exists(): + logger.warning(f"Device with name '{member_name}' already exists, skipping VC member creation") + continue + + Device.objects.create( + name=member_name, + device_type=master_device.device_type, + role=master_device.role, + site=master_device.site, + location=member_location, + rack=member_rack, + platform=master_device.platform, + serial=serial, + virtual_chassis=vc, + vc_position=chosen_pos, + comments=f"VC member (LibreNMS: {member.get('name', 'Unknown')})\n" + f"Auto-created from stack inventory", + ) + members_created += 1 + + # Validate member count + expected_members = len( + [m for m in members_info if not (m.get("serial") and m.get("serial") == master_device.serial)] + ) + if members_created < expected_members: + logger.warning( + f"Created {members_created} members but expected {expected_members}. " + "Some members may have been skipped due to duplicates." + ) + + logger.info( + f"Created Virtual Chassis '{vc.name}' with {vc.members.count()} total members " + f"(1 master + {members_created} additional)" + ) + + return vc + + except Exception as e: + # The transaction.atomic() block above will roll back all DB changes automatically. + # Manual state restoration is redundant and the save() would fail in a broken transaction. + logger.error( + f"Virtual Chassis creation failed for device {master_device.name}: {e}", + exc_info=True, + ) + raise diff --git a/netbox_librenms_plugin/import_utils/vm_operations.py b/netbox_librenms_plugin/import_utils/vm_operations.py new file mode 100644 index 0000000000..1272871a5a --- /dev/null +++ b/netbox_librenms_plugin/import_utils/vm_operations.py @@ -0,0 +1,229 @@ +"""Virtual machine creation and import operations.""" + +import logging + +from dcim.models import DeviceRole +from django.utils import timezone +from virtualization.models import Cluster + +from ..librenms_api import LibreNMSAPI +from .device_operations import _determine_device_name, fetch_device_with_cache, validate_device_for_import +from .permissions import require_permissions + +logger = logging.getLogger(__name__) + + +def create_vm_from_librenms( + libre_device: dict, validation: dict, use_sysname: bool = True, role=None, server_key: str = "default" +): + """ + Create a NetBox VirtualMachine from LibreNMS device data. + + Args: + libre_device: Device data from LibreNMS + validation: Validation result from validate_device_for_import with import_as_vm=True + use_sysname: If True, prefer sysName; if False, use hostname + role: Optional DeviceRole to assign to the VM + + Returns: + Created VirtualMachine instance + + Raises: + Exception if VM cannot be created + """ + from virtualization.models import VirtualMachine + + if not validation["can_import"]: + raise ValueError(f"VM cannot be imported: {', '.join(validation['issues'])}") + + # Extract matched objects from validation + cluster = validation["cluster"]["cluster"] + platform = validation["platform"].get("platform") + + # Determine VM name - use pre-computed name if available (handles strip_domain) + vm_name = libre_device.get("_computed_name") + if not vm_name: + vm_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=False, + device_id=libre_device.get("device_id"), + ) + + # Generate import timestamp comment + import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") + + # Create the VM with librenms_id custom field + vm = VirtualMachine.objects.create( + name=vm_name, + cluster=cluster, + role=role, # Optional VM role + platform=platform, + comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", + custom_field_data={"librenms_id": {server_key: int(libre_device["device_id"])}}, + ) + + logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") + return vm + + +def bulk_import_vms( + vm_imports: dict[int, dict[str, int]], + api: LibreNMSAPI, + sync_options: dict = None, + libre_devices_cache: dict = None, + job=None, + user=None, +) -> dict: + """ + Import multiple LibreNMS devices as VMs in NetBox. + + Handles validation, cluster/role assignment, name determination, + and VM creation. Supports both synchronous and background job execution. + + This function consolidates VM import logic that was previously duplicated + in BulkImportDevicesView and ImportDevicesJob, ensuring consistent behavior + across synchronous and background import paths. + + Args: + vm_imports: Dict mapping device_id to {"cluster_id": int, "device_role_id": int} + api: LibreNMSAPI instance for device fetching + sync_options: Optional dict with use_sysname, strip_domain settings + libre_devices_cache: Optional pre-fetched device data cache + job: Optional JobRunner instance for background job logging/cancellation + user: User performing the import (for permission checks). If job is provided, + user is extracted from job.job.user if not explicitly passed. + + Returns: + Dict with keys: + - success: List of {"device_id": int, "device": VM, "message": str} + - failed: List of {"device_id": int, "error": str} + - skipped: List of {"device_id": int, "reason": str} + + Raises: + PermissionDenied: If user lacks required permissions + + Example: + >>> # Synchronous import from view + >>> vm_imports = {123: {"cluster_id": 5, "device_role_id": 2}} + >>> result = bulk_import_vms(vm_imports, api, sync_options, user=request.user) + >>> print(f"Created {len(result['success'])} VMs") + >>> + >>> # Background job import + >>> result = bulk_import_vms(vm_imports, api, sync_options, cache, job=self) + """ + from netbox_librenms_plugin.import_validation_helpers import ( + apply_cluster_to_validation, + apply_role_to_validation, + ) + + # Extract user from job if not explicitly provided + if user is None and job is not None: + user = getattr(job.job, "user", None) + + # Check permissions at start of bulk operation + require_permissions(user, ["virtualization.add_virtualmachine"], "import VMs") + + result = {"success": [], "failed": [], "skipped": []} + vm_ids = list(vm_imports.keys()) + + # Use job logger if available, otherwise standard logger + log = job.logger if job else logger + + for idx, vm_id in enumerate(vm_ids, start=1): + # Check for job cancellation every 5 VMs + if job and idx % 5 == 0: + job.job.refresh_from_db() + job_status = job.job.status + status_value = job_status.value if hasattr(job_status, "value") else job_status + if status_value in ("failed", "errored"): + log.warning(f"Job cancelled at VM {idx} of {len(vm_ids)}") + break + log.info(f"Imported VM {idx} of {len(vm_ids)}") + + try: + # Fetch device data (uses cache helper) + libre_device = fetch_device_with_cache(vm_id, api, api.server_key, libre_devices_cache) + + if not libre_device: + result["failed"].append( + { + "device_id": vm_id, + "error": f"Device {vm_id} not found in LibreNMS", + } + ) + log.error(f"Device {vm_id} not found in LibreNMS") + continue + + # Validate as VM + use_sysname_opt = sync_options.get("use_sysname", True) if sync_options else True + strip_domain_opt = sync_options.get("strip_domain", False) if sync_options else False + validation = validate_device_for_import( + libre_device, + import_as_vm=True, + api=api, + use_sysname=use_sysname_opt, + strip_domain=strip_domain_opt, + server_key=api.server_key, + ) + + # Check if VM already exists + if validation.get("existing_device"): + result["skipped"].append( + { + "device_id": vm_id, + "reason": f"VM already exists: {validation['existing_device'].name}", + } + ) + log.info(f"VM already exists: {validation['existing_device'].name}") + continue + + # Apply manual cluster and role selections + vm_mappings = vm_imports[vm_id] + cluster_id = vm_mappings.get("cluster_id") + role_id = vm_mappings.get("device_role_id") + + if cluster_id: + cluster = Cluster.objects.filter(id=cluster_id).first() + if cluster: + apply_cluster_to_validation(validation, cluster) + + role = None + if role_id: + role = DeviceRole.objects.filter(id=role_id).first() + if role: + apply_role_to_validation(validation, role, is_vm=True) + + # Determine VM name + use_sysname = sync_options.get("use_sysname", True) if sync_options else True + strip_domain = sync_options.get("strip_domain", False) if sync_options else False + + vm_name = _determine_device_name( + libre_device, + use_sysname=use_sysname, + strip_domain=strip_domain, + device_id=vm_id, + ) + + # Update validation with computed name + libre_device["_computed_name"] = vm_name + + # Create VM + vm = create_vm_from_librenms( + libre_device, validation, use_sysname=use_sysname, role=role, server_key=api.server_key + ) + + result["success"].append( + { + "device_id": vm_id, + "device": vm, + "message": f"VM {vm.name} created successfully", + } + ) + log.info(f"Successfully imported VM {vm.name} (ID: {vm_id})") + + except Exception as vm_error: + log.error(f"Failed to import VM {vm_id}: {vm_error}", exc_info=True) + result["failed"].append({"device_id": vm_id, "error": str(vm_error)}) + + return result diff --git a/netbox_librenms_plugin/jobs.py b/netbox_librenms_plugin/jobs.py index bb3d00e37d..f476c1f7e7 100644 --- a/netbox_librenms_plugin/jobs.py +++ b/netbox_librenms_plugin/jobs.py @@ -46,6 +46,8 @@ def run( show_disabled, exclude_existing=False, server_key=None, + use_sysname=True, + strip_domain=False, **kwargs, ): """ @@ -60,6 +62,8 @@ def run( show_disabled: Whether to include disabled devices exclude_existing: Whether to exclude devices that already exist in NetBox server_key: Optional LibreNMS server key for multi-server setups + use_sysname: If True, prefer sysName over hostname + strip_domain: If True, strip domain suffix from device name **kwargs: Additional job parameters """ from netbox_librenms_plugin.import_utils import process_device_filters @@ -88,6 +92,8 @@ def run( show_disabled=show_disabled, exclude_existing=exclude_existing, job=self, + use_sysname=use_sysname, + strip_domain=strip_domain, ) # Store device IDs for result retrieval @@ -107,6 +113,8 @@ def run( "filters": filters, "server_key": server_key, "vc_detection_enabled": vc_detection_enabled, + "use_sysname": use_sysname, + "strip_domain": strip_domain, "cache_timeout": api.cache_timeout, "cached_at": cached_at, "completed": True, diff --git a/netbox_librenms_plugin/librenms_api.py b/netbox_librenms_plugin/librenms_api.py index 5de9db6c25..8e2b2b3ac0 100644 --- a/netbox_librenms_plugin/librenms_api.py +++ b/netbox_librenms_plugin/librenms_api.py @@ -190,7 +190,9 @@ def get_librenms_id(self, obj): If found via API, stores ID in custom field if available, otherwise caches the value. """ - librenms_id = obj.cf.get("librenms_id") + from netbox_librenms_plugin.utils import get_librenms_device_id + + librenms_id = get_librenms_device_id(obj, self.server_key) if librenms_id: return librenms_id @@ -254,7 +256,9 @@ def _store_librenms_id(self, obj, librenms_id): None """ if "librenms_id" in obj.cf: - obj.custom_field_data["librenms_id"] = librenms_id + from netbox_librenms_plugin.utils import set_librenms_device_id + + set_librenms_device_id(obj, librenms_id, self.server_key) obj.save() else: # Use cache as fallback diff --git a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js index c89473c02a..2e130b1284 100644 --- a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js +++ b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js @@ -310,6 +310,8 @@ * * @param {HTMLElement} modalElement - The modal element to hide * @param {Object} fallbackBackdropRef - Reference object containing fallback backdrop (deprecated) + * WONTFIX: fallbackBackdropRef is unused — _hideManual uses querySelector which is + * correct for this plugin since only one modal is ever open at a time (Tabler, no Bootstrap). */ function hideModal(modalElement, fallbackBackdropRef) { if (!modalElement) { @@ -317,6 +319,14 @@ } const manager = new ModalManager(modalElement); + + // Try to recover an existing Bootstrap instance before falling back to manual + if (typeof bootstrap !== 'undefined' && bootstrap.Modal) { + manager.instance = bootstrap.Modal.getInstance(modalElement); + } else if (typeof window.bootstrap !== 'undefined' && window.bootstrap.Modal) { + manager.instance = window.bootstrap.Modal.getInstance(modalElement); + } + manager.hide(); } @@ -338,6 +348,7 @@ function pollJobStatus(jobId, jobPk, pollUrl, baseUrl, originalFilters, deviceCount) { const messageEl = document.getElementById('filter-progress-message'); const cancelBtn = document.getElementById('cancel-filter-btn'); + const filterModal = document.getElementById('filter-processing-modal'); // Get CSRF token from cookie or form (needed for cancel and status sync) let csrfToken = getCookie('csrftoken'); @@ -402,11 +413,8 @@ messageEl.textContent = 'Job already completed, loading results...'; } cancelBtn.textContent = 'Completed'; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => { @@ -448,11 +456,8 @@ messageEl.textContent = 'Job cancelled successfully.'; } cancelBtn.textContent = 'Cancelled'; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => { @@ -469,11 +474,8 @@ messageEl.textContent = 'Job stopped (status sync failed).'; } cancelBtn.textContent = 'Stopped'; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => { @@ -486,11 +488,8 @@ messageEl.textContent = 'Job completed, loading results...'; } cancelBtn.textContent = 'Completed'; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => { @@ -505,11 +504,8 @@ } cancelBtn.textContent = 'Close'; cancelBtn.disabled = false; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => window.location.href = baseUrl, 1000); @@ -590,11 +586,8 @@ if (statusValue === 'completed' || statusValue === 'finished') { pollingStopped = true; // Stop future polls - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } // Small delay to let modal close before redirect @@ -604,21 +597,15 @@ return; // Stop polling } else if (statusValue === 'stopped') { pollingStopped = true; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } setTimeout(() => window.location.href = baseUrl, 100); } else if (statusValue === 'failed') { pollingStopped = true; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } const errorMsg = data.data?.error; @@ -628,11 +615,8 @@ setTimeout(() => window.location.href = baseUrl, 100); } else if (statusValue === 'errored') { pollingStopped = true; - - const modal = document.getElementById('filter-processing-modal'); - if (modal) { - const manager = new ModalManager(modal); - manager.hide(); + if (filterModal) { + hideModal(filterModal); } const errorMsg = data.data?.error || 'Job encountered an error. Please try again.'; @@ -1031,11 +1015,8 @@ if (failedCount && failedCount.dataset.failedCount === '0') { setTimeout(() => { const resultsModal = document.getElementById('import-results-modal'); - if (resultsModal && typeof bootstrap !== 'undefined' && bootstrap.Modal) { - const modalInstance = bootstrap.Modal.getInstance(resultsModal); - if (modalInstance) { - modalInstance.hide(); - } + if (resultsModal) { + hideModal(resultsModal); } window.location.reload(); }, MODAL_AUTO_CLOSE_MS); @@ -1113,6 +1094,12 @@ } showModal(modalElement, fallbackBackdropRef); + + // Re-initialize tooltips for newly swapped modal content + if (typeof bootstrap !== 'undefined' && bootstrap.Tooltip) { + const tooltipEls = modalContent.querySelectorAll('[data-bs-toggle="tooltip"]'); + [...tooltipEls].map(el => new bootstrap.Tooltip(el)); + } } document.body.addEventListener('htmx:afterSwap', ensureModalVisible); diff --git a/netbox_librenms_plugin/tables/device_status.py b/netbox_librenms_plugin/tables/device_status.py index 8bf978ccd4..3f03764cef 100644 --- a/netbox_librenms_plugin/tables/device_status.py +++ b/netbox_librenms_plugin/tables/device_status.py @@ -444,7 +444,7 @@ def render_actions(self, value, record): buttons = [] if existing: - # Link to existing device/VM in NetBox + # Link to existing device/VM in NetBox + details button for conflict resolution if isinstance(existing, VirtualMachine): url_name = "virtualization:virtualmachine" title = "View VM in NetBox" @@ -457,6 +457,50 @@ def render_actions(self, value, record): f'' ) + + # Add details/conflict button for conflict resolution actions + details_url = self._build_validation_details_url(device_id, validation) + match_type = validation.get("existing_match_type", "") + serial_action = validation.get("serial_action") + has_mismatch = validation.get("device_type_mismatch", False) + has_actions = match_type == "hostname" or (match_type == "serial" and serial_action is not None) + has_name_sync = validation.get("name_sync_available", False) + has_sync_needed = match_type == "librenms_id" and serial_action in ("update_serial", "conflict") + + if has_mismatch: + btn_class = "btn-outline-danger" + btn_icon = "mdi-alert-circle" + btn_label = " Conflict" + elif has_actions: + btn_class = "btn-outline-warning" + btn_icon = "mdi-alert" + btn_label = " Conflict" + elif has_name_sync or has_sync_needed: + btn_class = "btn-outline-warning" + btn_icon = "mdi-information-outline" + btn_label = " Details" + elif match_type == "librenms_id" and validation.get("librenms_id_needs_migration"): + btn_class = "btn-outline-warning" + btn_icon = "mdi-database-alert" + btn_label = " Legacy ID" + else: + btn_class = "btn-outline-success" + btn_icon = "mdi-check-circle" + btn_label = "" + + btn_title = "Resolve conflict" if (has_actions or has_mismatch) else "View details" + aria_attr = f'aria-label="{btn_title}" ' if btn_label == "" else "" + buttons.append( + f'' + ) elif is_ready: # Ready to import - show Import and Details buttons details_url = self._build_validation_details_url(device_id, validation) @@ -473,7 +517,7 @@ def render_actions(self, value, record): f'
- {# Device Summary #} -
-
- LibreNMS Device Information + {# Compute existing device URL once for use throughout #} + {% if validation.existing_device %} + {% if validation.import_as_vm or validation.existing_device.cluster %} + {% url 'virtualization:virtualmachine' pk=validation.existing_device.pk as existing_device_url %} + {% else %} + {% url 'dcim:device' pk=validation.existing_device.pk as existing_device_url %} + {% endif %} + {% endif %} + + {# Row: LibreNMS Status (left) + Device Info table (right) #} +
+ + {# Left: LibreNMS Status card #} +
+
+
LibreNMS Status
+
+ + + + + + + + + + + + + + + + + + + + + + + +
Status + {% if libre_device.status == 1 or libre_device.status == "1" %} + Up + {% elif libre_device.status == 0 or libre_device.status == "0" %} + Down + {% else %} + Unknown + {% endif %} +
Hostname{{ libre_device.hostname }}
ID{{ libre_device.device_id }}
IP{{ libre_device.ip|default:"—" }}
Location{{ libre_device.location|default:"—" }}
+
+
-
-
-
Hostname:
-
{{ libre_device.hostname }}
-
System Name:
-
{{ libre_device.sysName|default:"N/A" }}
+ {# Right: Device Information table #} +
+
+
+ Device Information + + {% if use_sysname %} + sysName + {% else %} + hostname + {% endif %} + {% if strip_domain %} + Domain stripped + {% endif %} + +
+
+ + + + + + + + + + {# Name row #} + + + + + -
LibreNMS ID:
-
{{ libre_device.device_id }}
+ {% if not validation.import_as_vm %} + {# Site row #} + + + + + -
IP Address:
-
{{ libre_device.ip|default:"N/A" }}
+ {# Device Type row #} + + + + + + {% endif %} -
Hardware:
-
{{ libre_device.hardware|default:"N/A" }}
+ {# Serial row (only for devices) #} + {% if not validation.import_as_vm %} + + + + + + {% endif %} -
OS:
-
{{ libre_device.os|default:"N/A" }}
+ {# Role row #} + + + + + -
Location:
-
{{ libre_device.location|default:"N/A" }}
+ {# Platform row #} + + + + + -
Status:
-
- {% if libre_device.status == 1 %} - Up - {% elif libre_device.status == 0 %} - Down - {% else %} - Unknown - {% endif %} -
- + {% if validation.import_as_vm %} + {# Cluster row (VMs only) #} + + + + + + {% else %} + {# Rack row #} + + + + + + {% endif %} + + {# Primary IP row #} + + + + + + +
FieldNetBox ValueLibreNMS Value
Name + {% if validation.existing_device %} + + {{ validation.existing_device.name }} + + {% if validation.name_sync_available %} +
+ {% csrf_token %} + + + +
+ {% endif %} + {% else %} + New device + {% endif %} + 		{{ libre_device.sysName|default:libre_device.hostname }}
Site + {% if validation.existing_device and validation.existing_device.site %} + {{ validation.existing_device.site }} + {% if validation.site.site and validation.existing_device.site.pk == validation.site.site.pk %} + + {% endif %} + {% elif validation.site.site %} + {{ validation.site.site.name }} + + {% else %} + No matching site + {% endif %} + {{ libre_device.location|default:"—" }}
Device Type + {% if validation.device_type_mismatch %} + + {{ validation.existing_device.device_type }} + + +
+ {% csrf_token %} + + + + +
+ {% elif validation.existing_device and validation.existing_device.device_type %} + {{ validation.existing_device.device_type }} + {% if sync_info and not sync_info.device_type_synced and sync_info.librenms_device_type %} +
+ {% csrf_token %} + + + +
+ {% elif validation.device_type.device_type %} + + {% endif %} + {% elif validation.device_type.device_type %} + + {{ validation.device_type.device_type }} + + + {% else %} + No matching type + {% endif %} + 		{{ libre_device.hardware|default:"—" }}
Serial + {% if validation.existing_device %} + {% if validation.existing_device.serial %} + {{ validation.existing_device.serial }} + {% else %} + Not set + {% endif %} + {% if sync_info and not sync_info.serial_synced %} + {% if validation.serial_action == 'conflict' %} + + + + {% else %} +
+ {% csrf_token %} + + + +
+ {% endif %} + {% elif sync_info and sync_info.serial_synced and sync_info.librenms_serial != '-' %} + + {% endif %} + {% else %} + — + {% endif %} + 		+ {{ libre_device.serial|default:"—" }} + {% if validation.serial_duplicate %} + Conflict + {% endif %} +
+ {% if validation.import_as_vm %}Role{% else %}Device Role{% endif %} + + {% if validation.existing_device and validation.existing_device.role %} + {{ validation.existing_device.role }} + + {% elif validation.device_role.role %} + {{ validation.device_role.role.name }} + + {% else %} + No role assigned + {% endif %} + —
Platform + {% if validation.existing_device and validation.existing_device.platform %} + {{ validation.existing_device.platform }} + {% if sync_info and not sync_info.platform_synced %} + {% if sync_info.platform_info.platform_exists %} + {% if not validation.import_as_vm and not validation.existing_device.cluster %} +
+ {% csrf_token %} + + + +
+ {% endif %} + {% else %} + + + + {% endif %} + {% elif sync_info and sync_info.platform_synced %} + + {% endif %} + {% elif validation.platform.platform %} + {{ validation.platform.platform.name }} + + {% elif sync_info and sync_info.platform_info.platform_exists %} + Not set + {% if validation.existing_device and not validation.import_as_vm and not validation.existing_device.cluster %} +
+ {% csrf_token %} + + + +
+ {% endif %} + {% else %} + Optional + {% endif %} + 		{{ libre_device.os|default:"—" }}
Cluster + {% if validation.cluster.cluster %} + {{ validation.cluster.cluster.name }} + + {% else %} + No cluster assigned + {% endif %} + —
Rack + {% if validation.rack.rack %} + {% if validation.rack.rack.location %} + {{ validation.rack.rack.location.name }} — {{ validation.rack.rack.name }} + {% else %} + {{ validation.rack.rack.name }} + {% endif %} + + {% else %} + Optional + {% endif %} + —
Primary IP + {% if validation.existing_device and validation.existing_device.primary_ip %} + {{ validation.existing_device.primary_ip }} + {% elif libre_device.ip %} + {{ libre_device.ip }} + {% else %} + No primary IP + {% endif %} + {{ libre_device.ip|default:"—" }}
+
+
- {# Validation Status #} -
-
- {% if validation.existing_device %} - {% if validation.existing_match_type == 'librenms_id' %} - {# Device was imported via this plugin #} -
-
- Device Already Imported -
-

- This device has already been imported to NetBox as - - {{ validation.existing_device.name }} - -

-

- - Matched by LibreNMS ID ({{ libre_device.device_id }}) - device is linked and synchronized. -

-
- {% elif validation.existing_match_type == 'hostname' %} - {# Device exists with same hostname but wasn't imported from LibreNMS #} -
-
- Device Exists (Hostname Match) -
-

- A device with the same hostname exists in NetBox as - - {{ validation.existing_device.name }} - -

-

- - Matched by hostname only - this device is not linked to LibreNMS. - If this is the same device, consider adding the LibreNMS ID custom field manually. -

-
- {% elif validation.existing_match_type == 'primary_ip' %} - {# Device exists with same IP but wasn't imported from LibreNMS #} -
-
- Device Exists (IP Match) -
-

- A device with the same IP address ({{ libre_device.ip }}) exists in NetBox as - - {{ validation.existing_device.name }} - -

-

- - Matched by IP address only - verify this is the same device. - If correct, consider adding the LibreNMS ID custom field manually. -

-
+ {# Status & Actions #} + {% if validation.existing_device %} + {% if validation.existing_match_type == 'librenms_id' %} +
+ {% if existing_id_servers %} + {% for srv in existing_id_servers %} + Linked — ID {{ srv.device_id }} @ {{ srv.display_name }} + {% endfor %} {% else %} - {# Fallback for unknown match type #} -
-
- Device Already Exists -
-

- This device already exists in NetBox as - - {{ validation.existing_device.name }} - -

-

- No import is needed. -

-
+ Linked — ID {{ libre_device.device_id }} {% endif %} - {% elif validation.is_ready %} -
-
- Ready to Import -
-

- All prerequisites are met. This device can be imported into NetBox without any issues. -

+ {% if validation.name_matches %} + Name match + {% elif validation.name_sync_available %} + Name differs + {% endif %} + {% if validation.serial_confirmed %} + Serial confirmed + {% elif validation.serial_action == 'conflict' %} + Serial conflict + {% elif validation.serial_action == 'update_serial' %} + Serial differs + {% endif %} + {% if validation.device_type_mismatch %} + Type mismatch + {% endif %} + {% if validation.librenms_id_needs_migration %} + Legacy ID format + {% endif %} +
+ {% if validation.librenms_id_needs_migration %} +
+
+ {% csrf_token %} + + + {% if not validation.serial_confirmed %} +
+ + +
+ {% endif %} + +
+
+ {% endif %} + + {% elif validation.existing_match_type == 'hostname' %} +
+ Hostname match + {% if validation.serial_confirmed %} + Serial confirmed + {% elif validation.serial_action == 'conflict' %} + Serial conflict + {% elif validation.serial_action == 'update_serial' %} + Serial differs + {% endif %} + {% if validation.device_type_mismatch %} + Type mismatch + {% endif %} + + — Exists as + {{ validation.existing_device.name }}, + not linked to LibreNMS. + +
+ {% if validation.serial_action == 'conflict' %} +
+ + Import blocked: The incoming serial number is already assigned to another device in NetBox. + Resolve the duplicate serial before linking.
- {% elif validation.can_import %} -
-
- Can Import with Warnings -
-

- This device can be imported, but there are some warnings to review below. -

+ {% elif validation.import_as_vm %} +
+ + Hostname match found for a VM — use the import action to proceed.
{% else %} -
-
- Cannot Import -
-

- This device cannot be imported due to validation issues listed below. -

-
- {% endif %} -
-
- - {# Validation Details #} -
-
- NetBox Import Matching -
-
-

- - The following NetBox objects will be used when importing this - {% if validation.import_as_vm %}Virtual Machine{% else %}Device{% endif %}: - -

- - - {% if validation.import_as_vm %} - {# VM-specific fields #} - - - - - - - - - - - - - - - +
+
+ {% csrf_token %} + + {% if validation.device_type_mismatch %} +
+ + +
+ {% endif %} + {% if validation.serial_action == 'update_serial' %} + + {% else %} - {# Device-specific fields #} -
- - - - - - - - - - - - - + + + {% endif %} + + + {% endif %} - - - - + {% elif validation.existing_match_type == 'serial' %} +
+ Serial match + {% if validation.serial_action == 'hostname_differs' %} + Name differs + {% elif validation.serial_action == 'link' %} + Name match + {% endif %} + {% if validation.device_type_mismatch %} + Type mismatch + {% endif %} + + — Exists as + {{ validation.existing_device.name }}, + not linked to LibreNMS. + +
+
+
+ {% csrf_token %} + + {% if validation.device_type_mismatch %} +
+ + +
+ {% endif %} + {% if validation.serial_action == 'link' %} + + + {% elif validation.serial_action == 'hostname_differs' %} + {% endif %} + +
- {# Common fields for both Device and VM #} - - - - + {% elif validation.existing_match_type == 'primary_ip' %} +
+ IP match + + — Device with IP {{ libre_device.ip }} exists as + {{ validation.existing_device.name }}. + Consider adding LibreNMS ID manually. + +
- {% if not validation.import_as_vm %} - - - - - {% endif %} + {% else %} +
+ + Exists — Device already exists as + {{ validation.existing_device.name }}. +
+ {% endif %} - - - - - -
Import Type: - - Virtual Machine -
Cluster: - {% if validation.cluster.cluster %} - - {{ validation.cluster.cluster.name }} - {% else %} - - No cluster assigned (required for VMs) - {% endif %} -
Role: - {% if validation.device_role.role %} - - {{ validation.device_role.role.name }} - {% else %} - - Optional - not assigned - {% endif %} -
Import Type: - - Device -
Site: - {% if validation.site.site %} - - {{ validation.site.site.name }} - {% else %} - - No matching site found - {% endif %} -
Device Type: - {% if validation.device_type.device_type %} - - {{ validation.device_type.device_type }} - {% else %} - - No matching device type found - {% endif %} -
Device Role: - {% if validation.device_role.role %} - - {{ validation.device_role.role.name }} - {% else %} - - No device role assigned - {% endif %} -
Platform: - {% if validation.platform.platform %} - - {{ validation.platform.platform.name }} - {% else %} - - Optional - will be auto-mapped if available - {% endif %} -
Rack: - {% if validation.rack.rack %} - - - {% if validation.rack.rack.location %} - {{ validation.rack.rack.location.name }} - {{ validation.rack.rack.name }} - {% else %} - {{ validation.rack.rack.name }} - {% endif %} - - {% else %} - - Optional - not assigned - {% endif %} -
Primary IP: - {% if libre_device.ip %} - - {{ libre_device.ip }} - {% else %} - - No primary IP - will import without - {% endif %} -
+ {% elif validation.is_ready %} +
+ + Ready to Import — All prerequisites are met.
-
+ {% elif validation.can_import %} +
+ + Can Import — Review warnings below before importing. +
+ {% else %} +
+ + Cannot Import — Validation issues prevent import. +
+ {% endif %} {# Warnings #} {% if validation.warnings %}
-
+
Warnings ({{ validation.warnings|length }})
-
-

These warnings won't prevent import but should be reviewed:

-
    +
    +
      {% for warning in validation.warnings %} -
    • {{ warning }}
      • +
    • {{ warning }}
      • {% endfor %}
    @@ -331,23 +590,24 @@
    {% if validation.existing_device %} {% if validation.import_as_vm or validation.existing_device.cluster %} - {# It's a VM #} - + View VM in NetBox {% else %} - {# It's a Device #} - - View Device in NetBox + + View in NetBox + {% if validation.existing_match_type == 'librenms_id' %} + + Full Sync Page + + {% endif %} {% endif %} {% endif %} - -
    diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_import.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_import.html index 0013ec5be3..f5c0eeaf49 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_import.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_import.html @@ -399,7 +399,7 @@
    {# HTMX Modal for import actions #}