From 5d4392cd4bda944740194593115dd4b470ad1ee2 Mon Sep 17 00:00:00 2001 From: Andy Norwood <2754635+bonzo81@users.noreply.github.com> Date: Thu, 19 Feb 2026 16:03:52 +0000 Subject: [PATCH 01/28] Create pull request template for contributions Added a pull request template to standardize PR submissions. --- .github/pull_request_template.md | 49 ++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000000..6d026f4f9e --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,49 @@ +## Summary +Briefly describe what this PR does in plain English, and provide as much of the following information as possible. + +## Motivation / Problem +What issue does this solve? +- Bug +- Feature +- Refactor +- Maintenance / cleanup + +Link any related issues if applicable. + +## Scope of Change +Tick all that apply: + +- [ ] Sync/Import logic +- [ ] NetBox models / ORM +- [ ] LibreNMS API interaction +- [ ] Config / settings +- [ ] Web UI / templates +- [ ] Database migrations +- [ ] Tests +- [ ] Docs only +- Other (please descibe)? + +## How Was This Tested? +Tick all that apply and describe briefly. + +- [ ] Unit tests +- [ ] Manual testing +- [ ] Not tested (explain why) + +### Manual Test Steps (if applicable) +1. +2. +3. + +## Risk Assessment +- Does this change affect existing users? +- Could this cause unintended imports / updates? + +Explain briefly. + +## Backwards Compatibility +- [ ] No breaking changes +- [ ] Breaking change (explain and document) + +## Other Notes +Anything the maintainer(s) should pay particular attention to? From b14cf4a836b26735de36b127b91eb9a7097bff03 Mon Sep 17 00:00:00 2001 From: Andy Norwood <2754635+bonzo81@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:03:29 +0000 Subject: [PATCH 02/28] Remove checkbox to avoid task creation --- .github/pull_request_template.md | 34 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6d026f4f9e..b641e68a71 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -11,24 +11,24 @@ What issue does this solve? Link any related issues if applicable. ## Scope of Change -Tick all that apply: - -- [ ] Sync/Import logic -- [ ] NetBox models / ORM -- [ ] LibreNMS API interaction -- [ ] Config / settings -- [ ] Web UI / templates -- [ ] Database migrations -- [ ] Tests -- [ ] Docs only -- Other (please descibe)? +Delete items that don’t apply: + +- Sync/Import logic +- NetBox models / ORM +- LibreNMS API interaction +- Config / settings +- Web UI / templates +- Database migrations +- Tests +- Docs only +- Other: ## How Was This Tested? -Tick all that apply and describe briefly. +Delete items that don’t apply and describe briefly. -- [ ] Unit tests -- [ ] Manual testing -- [ ] Not tested (explain why) +- Unit tests: +- Manual testing: +- Not tested: ### Manual Test Steps (if applicable) 1. @@ -42,8 +42,8 @@ Tick all that apply and describe briefly. Explain briefly. ## Backwards Compatibility -- [ ] No breaking changes -- [ ] Breaking change (explain and document) +- No breaking changes +- Breaking change (explain and document) ## Other Notes Anything the maintainer(s) should pay particular attention to? From a9ad37ab41efe14dc2300a56f542e2a66e6e008c Mon Sep 17 00:00:00 2001 From: Andy Norwood <2754635+bonzo81@users.noreply.github.com> Date: Tue, 3 Mar 2026 13:33:21 +0000 Subject: [PATCH 03/28] Update supported NetBox versions in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f8961beff..80969d1723 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ Alternatively, share your ideas for the plugin over in [discussions](https://git | NetBox Version | Plugin Version | |----------------|----------------| | 4.1 | 0.2.x - 0.3.5 | -| 4.2 - 4.4 | 0.3.6+ | +| 4.2 - 4.5 | 0.3.6+ | ## Installing From 62f838cf3df6ccdab73eabe7697097ca9b7b4665 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Wed, 4 Mar 2026 14:48:05 +0100 Subject: [PATCH 04/28] feat: migrate librenms_id to JSON multi-server format - Store librenms_id as {server_key: device_id} dict instead of bare int - Add get_librenms_device_id/set_librenms_device_id/find_by_librenms_id/ migrate_legacy_librenms_id helpers in utils.py (at end of file) - Thread server_key through import pipeline (filters, cache, bulk_import, device_operations, vm_operations, virtual_chassis) - Deterministic SHA256 cache keys, None-safe filter inclusion - Fix disabled field filter (use 'disabled' flag, not 'status') - Truthy string parsing for use_sysname/strip_domain ('on'/'true'/'1') - migrate_librenms_id action in DeviceConflictActionView - RemoveServerMappingView for per-server librenms_id removal - New tests: test_permissions.py, test_sync_view_mismatch.py - RQ-based job cancellation check in bulk import Reduce cosmetic diff vs develop: - _save_device restored to before _resolve_naming_preferences in actions.py - _empty_return moved to just before process_device_filters in bulk_import.py - New utils.py helpers at end of file with imports merged into main block --- netbox_librenms_plugin/forms.py | 4 +- .../import_utils/bulk_import.py | 222 +- netbox_librenms_plugin/import_utils/cache.py | 54 +- .../import_utils/device_operations.py | 114 +- .../import_utils/filters.py | 29 +- .../import_utils/virtual_chassis.py | 104 +- .../import_utils/vm_operations.py | 13 +- netbox_librenms_plugin/jobs.py | 2 + netbox_librenms_plugin/librenms_api.py | 8 +- .../js/librenms_import.js | 6 + .../tables/device_status.py | 6 + netbox_librenms_plugin/tables/interfaces.py | 6 +- .../htmx/device_validation_details.html | 76 +- .../librenms_sync_base.html | 64 +- .../tests/test_background_jobs.py | 10 + .../tests/test_import_utils.py | 2074 ++++++++++++----- .../tests/test_permissions.py | 127 + .../tests/test_sync_view_mismatch.py | 109 + netbox_librenms_plugin/urls.py | 6 + netbox_librenms_plugin/utils.py | 110 + netbox_librenms_plugin/views/__init__.py | 1 + .../views/base/cables_view.py | 29 +- .../views/base/ip_addresses_view.py | 7 +- .../views/base/librenms_sync_view.py | 50 + .../views/imports/actions.py | 363 ++- netbox_librenms_plugin/views/imports/list.py | 10 +- .../views/object_sync/devices.py | 13 +- .../views/object_sync/vms.py | 4 +- .../views/sync/device_fields.py | 122 +- .../views/sync/interfaces.py | 8 +- uv.lock | 8 + 31 files changed, 2774 insertions(+), 985 deletions(-) create mode 100644 uv.lock diff --git a/netbox_librenms_plugin/forms.py b/netbox_librenms_plugin/forms.py index f3e3d075e5..d33e94d4ca 100644 --- a/netbox_librenms_plugin/forms.py +++ b/netbox_librenms_plugin/forms.py @@ -581,7 +581,8 @@ def _populate_librenms_locations(self): try: # Use caching to avoid repeated API calls - cache_key = "librenms_locations_choices" + api = LibreNMSAPI() + cache_key = f"librenms_locations_choices:{api.server_key}" cached_choices = cache.get(cache_key) if cached_choices: @@ -589,7 +590,6 @@ def _populate_librenms_locations(self): return # Fetch locations from LibreNMS - api = LibreNMSAPI() success, locations = api.get_locations() if success and locations: diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 5e98347efe..845ceca0c1 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -1,4 +1,4 @@ -"""Bulk import orchestration and filter processing.""" +"""Bulk import orchestration for devices and filter processing.""" import logging from typing import List @@ -7,6 +7,7 @@ from django.core.cache import cache from ..librenms_api import LibreNMSAPI +from ..utils import find_by_librenms_id from .cache import get_cache_metadata_key, get_import_device_cache_key, get_validated_device_cache_key from .device_operations import import_single_device, validate_device_for_import from .filters import get_librenms_devices_for_import @@ -89,21 +90,34 @@ def bulk_import_devices_shared( api = LibreNMSAPI(server_key=server_key) for idx, device_id in enumerate(device_ids, start=1): - # Check for job cancellation every 5 devices - if job and idx % 5 == 0: - # Refresh job from DB to get current status - job.job.refresh_from_db() - job_status = job.job.status - status_value = job_status.value if hasattr(job_status, "value") else job_status - if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): - if job.logger: - job.logger.warning(f"Import job cancelled at device {idx} of {total}") - else: - logger.warning(f"Import cancelled at device {idx} of {total}") - break - # Log progress - if job.logger: - job.logger.info(f"Imported device {idx} of {total}") + # Check for job cancellation on first iteration and every 5th thereafter. + # Check RQ/Redis state first (reflects stop API immediately); fall back to DB. + if job and (idx == 1 or idx % 5 == 0): + try: + from django_rq import get_queue + from rq.job import Job as RQJob + + queue = get_queue("default") + rq_job = RQJob.fetch(str(job.job.job_id), connection=queue.connection) + if rq_job.is_failed or rq_job.is_stopped: + if job.logger: + job.logger.warning( + f"Import job stopped at device {idx} of {total} (RQ status: {rq_job.get_status()})" + ) + else: + logger.warning(f"Import cancelled at device {idx} of {total}") + break + except Exception: + # Fall back to DB check if RQ is unavailable + job.job.refresh_from_db() + job_status = job.job.status + status_value = job_status.value if hasattr(job_status, "value") else job_status + if status_value in (JobStatusChoices.STATUS_FAILED, "failed", "errored"): + if job.logger: + job.logger.warning(f"Import job cancelled at device {idx} of {total}") + else: + logger.warning(f"Import cancelled at device {idx} of {total}") + break try: # Use cached device data if available to avoid redundant API calls @@ -129,6 +143,7 @@ def bulk_import_devices_shared( api=api, use_sysname=use_sysname_opt, strip_domain=strip_domain_opt, + server_key=api.server_key, ) # Build manual mappings from validation + any provided overrides @@ -148,7 +163,7 @@ def bulk_import_devices_shared( result = import_single_device( device_id, - server_key=server_key, + server_key=api.server_key, # use resolved key, not raw parameter (may be None) sync_options=sync_options, manual_mappings=device_mappings if device_mappings else None, libre_device=libre_device, @@ -162,11 +177,22 @@ def bulk_import_devices_shared( "message": result["message"], } ) + # Log progress after each successful import + if job and job.logger: + job.logger.info(f"Imported device {idx} of {total}") # Handle virtual chassis creation for stacks vc_data = validation.get("virtual_chassis", {}) if vc_data.get("is_stack", False): - vc_domain = f"librenms-{device_id}" + # Derive a stack-level dedup key from member serials so that all + # LibreNMS devices belonging to the same physical stack (e.g. each + # switch in a stacked chassis that appears as a separate device in + # LibreNMS) share the same key and VC creation is triggered only once. + # Fall back to device_id when no member serials are available. + member_serials = sorted(m.get("serial") for m in vc_data.get("members", []) if m.get("serial")) + vc_domain = ( + f"librenms-stack-{','.join(member_serials)}" if member_serials else f"librenms-{device_id}" + ) # Only create VC if we haven't processed this stack yet # Add to set BEFORE attempting creation to prevent race condition @@ -266,45 +292,98 @@ def bulk_import_devices( ) -def _refresh_existing_device(validation: dict) -> None: - """Refresh existing_device from DB to pick up changes made in NetBox since caching.""" +def _refresh_existing_device(validation: dict, libre_device: dict = None, server_key: str = "default") -> None: + """Refresh existing_device from DB to pick up changes made in NetBox since caching. + + When existing_device is None (wasn't found at cache time), re-check if the device + was imported since caching by looking up librenms_id or hostname. + """ existing = validation.get("existing_device") - if not existing or not hasattr(existing, "pk"): + if existing and hasattr(existing, "pk"): + try: + from dcim.models import Device + from virtualization.models import VirtualMachine + + if validation.get("import_as_vm"): + refreshed = VirtualMachine.objects.filter(pk=existing.pk).first() + else: + refreshed = Device.objects.filter(pk=existing.pk).first() + + if refreshed: + validation["existing_device"] = refreshed + if hasattr(refreshed, "role") and refreshed.role: + validation["device_role"] = {"found": True, "role": refreshed.role} + else: + # Device was deleted since caching — recompute readiness to match + # validate_device_for_import logic. + validation["existing_device"] = None + validation["existing_match_type"] = None + can_import = not bool(validation.get("issues")) + if validation.get("import_as_vm"): + # VMs only require a cluster (site/role not mandatory) + is_ready = can_import and bool(validation.get("cluster", {}).get("found")) + else: + is_ready = ( + can_import + and bool(validation.get("site", {}).get("found")) + and bool(validation.get("device_type", {}).get("found")) + and bool(validation.get("device_role", {}).get("found")) + ) + validation["can_import"] = can_import + validation["is_ready"] = is_ready + except Exception as e: + existing_id = getattr(existing, "pk", "unknown") if existing else "none" + logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") + return + + # existing_device was None at cache time — check if device was imported since + if not libre_device: return try: from dcim.models import Device from virtualization.models import VirtualMachine - if validation.get("import_as_vm"): - refreshed = VirtualMachine.objects.filter(pk=existing.pk).first() - else: - refreshed = Device.objects.filter(pk=existing.pk).first() - - if refreshed: - validation["existing_device"] = refreshed - if hasattr(refreshed, "role") and refreshed.role: - validation["device_role"]["found"] = True - validation["device_role"]["role"] = refreshed.role - else: - # Device was deleted since caching — recompute readiness - validation["existing_device"] = None - validation["existing_match_type"] = None - if validation.get("import_as_vm"): - required_found = ( - validation.get("site", {}).get("found") - and validation.get("cluster", {}).get("found") - and validation.get("device_role", {}).get("found") - ) - else: - required_found = ( - validation.get("site", {}).get("found") - and validation.get("device_type", {}).get("found") - and validation.get("device_role", {}).get("found") - ) - validation["can_import"] = validation["is_ready"] = bool(required_found and not validation.get("issues")) + import_as_vm = validation.get("import_as_vm", False) + Model = VirtualMachine if import_as_vm else Device + + librenms_id = libre_device.get("device_id") + hostname = libre_device.get("hostname", "") + sys_name = libre_device.get("sysName", "") + + new_device = None + match_type = None + + # Check by librenms_id custom field first (JSON multi-server format + legacy) + if librenms_id: + try: + new_device = find_by_librenms_id(Model, int(librenms_id), server_key) + if new_device: + match_type = "librenms_id" + except (ValueError, TypeError): + pass + + # Fall back to hostname match, then sys_name independently + if not new_device and hostname: + new_device = Model.objects.filter(name__iexact=hostname).first() + if new_device: + match_type = "hostname" + if not new_device and sys_name: + new_device = Model.objects.filter(name__iexact=sys_name).first() + if new_device: + match_type = "hostname" + + if new_device: + validation["existing_device"] = new_device + validation["existing_match_type"] = match_type + validation["can_import"] = False + validation["is_ready"] = False except Exception as e: - existing_id = getattr(existing, "pk", "unknown") if existing else "none" - logger.error(f"Failed to refresh existing device (pk={existing_id}): {e}") + logger.error(f"Failed to re-check for imported device: {e}") + + +def _empty_return(return_cache_status: bool): + """Centralised empty-result return value for process_device_filters.""" + return ([], False) if return_cache_status else [] def process_device_filters( @@ -338,7 +417,7 @@ def process_device_filters( request: Optional Django request for client disconnect detection (synchronous only) return_cache_status: When True, returns (devices, from_cache) tuple use_sysname: If True, prefer sysName over hostname for device name resolution - strip_domain: If True, strip domain suffix from device names + strip_domain: If True, strip domain suffix from device name Returns: List[dict]: Validated devices with _validation key, or tuple of (devices, from_cache) @@ -360,9 +439,11 @@ def process_device_filters( return_cache_status=True, ) - # Filter out disabled devices if requested + # Filter out disabled devices if requested. LibreNMS's "disabled" field (1=disabled, + # 0=enabled) reflects manual device disablement; "status" reflects SNMP reachability. + # show_disabled controls the former: hidden when disabled==1, shown regardless of status. if not show_disabled: - libre_devices = [d for d in libre_devices if d.get("status") == 1] + libre_devices = [d for d in libre_devices if int(d.get("disabled", 0)) != 1] if job: job.logger.info(f"Found {len(libre_devices)} devices to process") @@ -386,7 +467,7 @@ def process_device_filters( except (BrokenPipeError, ConnectionError, IOError) as e: if request: logger.info(f"Client disconnected during VC prefetch: {e}") - return [] + return _empty_return(return_cache_status) raise # Validate each device @@ -406,13 +487,13 @@ def process_device_filters( if rq_job.is_failed or rq_job.is_stopped: job.logger.warning("Job was already stopped before validation started") - return [] + return _empty_return(return_cache_status) except Exception: # Fall back to DB check if RQ check fails job.job.refresh_from_db() - if job.job.status == JobStatusChoices.STATUS_FAILED: + if job.job.status in (JobStatusChoices.STATUS_FAILED, JobStatusChoices.STATUS_ERRORED): job.logger.warning("Job was stopped before validation started") - return [] + return _empty_return(return_cache_status) else: logger.info(f"Validating {total} devices") @@ -435,21 +516,13 @@ def process_device_filters( job.logger.info( f"Job stopped at device {idx}/{total} (RQ status: {rq_job.get_status()}). Exiting gracefully." ) - return [] + return _empty_return(return_cache_status) except Exception: # If we can't check RQ status, fall back to DB status check job.job.refresh_from_db() - if job.job.status == JobStatusChoices.STATUS_FAILED: + if job.job.status in (JobStatusChoices.STATUS_FAILED, JobStatusChoices.STATUS_ERRORED): job.logger.info(f"Job stopped at device {idx}/{total}. Exiting gracefully.") - return [] - elif request: - # Check for client disconnect - try: - if hasattr(request, "META") and request.META.get("wsgi.input"): - pass - except (BrokenPipeError, ConnectionError, IOError): - logger.info(f"Client disconnected during validation at device {idx}") - return [] + return _empty_return(return_cache_status) # Drop any cached validation/meta keys before recomputing device.pop("_validation", None) @@ -461,6 +534,8 @@ def process_device_filters( filters=filters, device_id=device_id, vc_enabled=vc_detection_enabled, + use_sysname=use_sysname, + strip_domain=strip_domain, ) # Check if we already have cached validation for this device @@ -473,7 +548,7 @@ def process_device_filters( # Refresh existing_device from DB to avoid stale data # (user may have changed role, name, etc. in NetBox) - _refresh_existing_device(device["_validation"]) + _refresh_existing_device(device["_validation"], libre_device=device, server_key=api.server_key) # Apply exclude_existing filter if enabled if exclude_existing: @@ -491,13 +566,14 @@ def process_device_filters( api=api_for_validation, include_vc_detection=vc_detection_enabled, force_vc_refresh=clear_cache, + server_key=api.server_key, use_sysname=use_sysname, strip_domain=strip_domain, ) except (BrokenPipeError, ConnectionError, IOError) as e: if request: logger.info(f"Client disconnected during device validation: {e}") - return [] + return _empty_return(return_cache_status) raise # Set VC detection metadata @@ -531,7 +607,11 @@ def process_device_filters( from datetime import datetime, timezone cache_metadata_key = get_cache_metadata_key( - server_key=api.server_key, filters=filters, vc_enabled=vc_detection_enabled + server_key=api.server_key, + filters=filters, + vc_enabled=vc_detection_enabled, + use_sysname=use_sysname, + strip_domain=strip_domain, ) # Check if metadata already exists to preserve original timestamp diff --git a/netbox_librenms_plugin/import_utils/cache.py b/netbox_librenms_plugin/import_utils/cache.py index fe0896a302..140f45f79d 100644 --- a/netbox_librenms_plugin/import_utils/cache.py +++ b/netbox_librenms_plugin/import_utils/cache.py @@ -1,5 +1,7 @@ -"""Cache key generation and search management for device import operations.""" +"""Cache key generation and management for device import operations.""" +import hashlib +import json import logging from django.core.cache import cache @@ -7,7 +9,9 @@ logger = logging.getLogger(__name__) -def get_cache_metadata_key(server_key: str, filters: dict, vc_enabled: bool) -> str: +def get_cache_metadata_key( + server_key: str, filters: dict, vc_enabled: bool, use_sysname: bool = True, strip_domain: bool = False +) -> str: """ Generate a consistent cache metadata key from filter parameters. @@ -15,13 +19,16 @@ def get_cache_metadata_key(server_key: str, filters: dict, vc_enabled: bool) -> server_key: LibreNMS server identifier filters: Filter dictionary vc_enabled: Whether VC detection is enabled + use_sysname: Whether sysName is preferred over hostname for device naming + strip_domain: Whether domain suffix is stripped from device names Returns: str: Consistent cache key for metadata """ - # Sort filter items to ensure consistent key generation - filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v) - return f"librenms_filter_cache_metadata_{server_key}_{filter_parts}_{vc_enabled}" + # Sort filter items to ensure consistent key generation; use "is not None" to preserve + # valid falsy values like 0 and False (filtering only None/missing entries). + filter_parts = "_".join(f"{k}={v}" for k, v in sorted(filters.items()) if v is not None) + return f"librenms_filter_cache_metadata_{server_key}_{filter_parts}_{vc_enabled}_sysname={use_sysname}_strip={strip_domain}" def get_active_cached_searches(server_key: str) -> list[dict]: @@ -61,8 +68,9 @@ def get_active_cached_searches(server_key: str) -> list[dict]: "other": "Other", } - # Get cached location choices for enrichment - location_cache_key = "librenms_locations_choices" + # Get cached location choices for enrichment; scoped by server_key so labels + # from different LibreNMS servers don't bleed into each other's filter summaries. + location_cache_key = f"librenms_locations_choices:{server_key}" cached_locations = cache.get(location_cache_key) if cached_locations: location_choices = dict(cached_locations) @@ -71,9 +79,18 @@ def get_active_cached_searches(server_key: str) -> list[dict]: metadata = cache.get(cache_key) if metadata: # Cache still exists, calculate time remaining - cached_at = datetime.fromisoformat(metadata.get("cached_at")) cache_timeout = metadata.get("cache_timeout", 300) now = datetime.now(timezone.utc) + try: + cached_at_raw = metadata.get("cached_at") + cached_at = ( + datetime.fromisoformat(cached_at_raw) if cached_at_raw else datetime.fromtimestamp(0, timezone.utc) + ) + # Normalize naive datetimes (e.g., stored without tzinfo) to UTC + if cached_at.tzinfo is None: + cached_at = cached_at.replace(tzinfo=timezone.utc) + except (ValueError, TypeError): + cached_at = datetime.fromtimestamp(0, timezone.utc) age_seconds = (now - cached_at).total_seconds() remaining_seconds = max(0, cache_timeout - age_seconds) @@ -109,7 +126,14 @@ def get_active_cached_searches(server_key: str) -> list[dict]: return active_searches -def get_validated_device_cache_key(server_key: str, filters: dict, device_id: int | str, vc_enabled: bool) -> str: +def get_validated_device_cache_key( + server_key: str, + filters: dict, + device_id: int | str, + vc_enabled: bool, + use_sysname: bool = True, + strip_domain: bool = False, +) -> str: """ Generate a consistent cache key for validated device data. @@ -121,6 +145,8 @@ def get_validated_device_cache_key(server_key: str, filters: dict, device_id: in filters: Filter dict with location, type, os, hostname, sysname, hardware keys device_id: LibreNMS device ID vc_enabled: Whether virtual chassis detection was enabled + use_sysname: Whether sysName is preferred over hostname for device naming + strip_domain: Whether domain suffix is stripped from device names Returns: str: Cache key for the validated device @@ -128,12 +154,14 @@ def get_validated_device_cache_key(server_key: str, filters: dict, device_id: in Example: >>> key = get_validated_device_cache_key('default', {'location': 'NYC'}, 123, True) >>> key - 'validated_device_default_-1234567890_123_vc' + 'validated_device_default_e3b0c44298fc1c14_123_vc' """ - # Sort filters for consistent hashing - filter_hash = hash(str(sorted(filters.items()))) + # Sort filters for a deterministic, cross-process stable hash + filter_hash = hashlib.sha256(json.dumps(sorted(filters.items()), sort_keys=True).encode()).hexdigest()[:16] vc_part = "vc" if vc_enabled else "novc" - return f"validated_device_{server_key}_{filter_hash}_{device_id}_{vc_part}" + return ( + f"validated_device_{server_key}_{filter_hash}_{device_id}_{vc_part}_sysname={use_sysname}_strip={strip_domain}" + ) def get_import_device_cache_key(device_id: int | str, server_key: str = "default") -> str: diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index 68c221eacf..0420b87447 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -1,12 +1,13 @@ -"""Device validation, import, and matching operations.""" +"""Device validation, import, and fetch operations.""" import logging from dcim.models import Device, DeviceRole, DeviceType, Rack, Site from django.core.cache import cache from django.db import transaction +from django.db.models import Q from django.utils import timezone -from virtualization.models import Cluster +from virtualization.models import Cluster # noqa: F401 — used by test mock.patch targets from ..librenms_api import LibreNMSAPI from ..utils import ( @@ -90,6 +91,7 @@ def validate_device_for_import( force_vc_refresh: bool = False, use_sysname: bool = True, strip_domain: bool = False, + server_key: str = "default", ) -> dict: """ Validate if a LibreNMS device can be imported to NetBox. @@ -164,6 +166,7 @@ def validate_device_for_import( "serial_action": None, # None, "link", "conflict", "update_serial", "hostname_differs" "serial_confirmed": False, # True when librenms_id match and serial matches "serial_duplicate": False, # True when incoming serial is already on a different device + "librenms_id_needs_migration": False, # True when librenms_id is still a legacy bare int "name_matches": False, # True when existing device name matches LibreNMS sysName "name_sync_available": False, # True when existing device name differs from sysName "suggested_name": None, # sysName to suggest when name_sync_available is True @@ -199,6 +202,7 @@ def validate_device_for_import( "rack": None, "available_racks": [], }, + "naming_criteria": None, # Populated after resolved_name is set } try: @@ -217,7 +221,7 @@ def validate_device_for_import( "strip_domain": strip_domain, "raw_sysname": libre_device.get("sysName") or "", "raw_hostname": libre_device.get("hostname") or "", - "source": "sysName" if use_sysname else "hostname", + "source": "sysname" if use_sysname and libre_device.get("sysName") else "hostname", } logger.debug( f"Checking for existing device/VM: " @@ -228,9 +232,10 @@ def validate_device_for_import( from virtualization.models import VirtualMachine # Check for existing VM first (by librenms_id custom field) - # Always query with int to match custom field type try: - existing_vm = VirtualMachine.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() + from netbox_librenms_plugin.utils import find_by_librenms_id + + existing_vm = find_by_librenms_id(VirtualMachine, int(librenms_id), server_key) except (ValueError, TypeError): # librenms_id is not convertible to int; no match will be found existing_vm = None @@ -242,7 +247,14 @@ def validate_device_for_import( result["import_as_vm"] = True # Force VM mode since VM exists result["can_import"] = False - # Check if name matches sysName + # Detect legacy bare-integer format so UI can offer a migration action. + # Direct access needed to detect legacy integer format for migration prompt: + # LibreNMSAPI.get_librenms_id() returns an int in both formats, so only the + # raw type check on custom_field_data reveals whether migration is needed. + if isinstance(existing_vm.custom_field_data.get("librenms_id"), int): + result["librenms_id_needs_migration"] = True + + # Check if name matches resolved name (accounts for use_sysname/strip_domain) # Note: name_sync_available/suggested_name are intentionally not set for VMs # because UpdateDeviceNameView only supports Device objects; VM name-sync # would require a separate implementation. @@ -250,10 +262,11 @@ def validate_device_for_import( result["name_matches"] = True # Check for existing Device (by librenms_id custom field) - # Always query with int to match custom field type if not result["existing_device"]: try: - existing_device = Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)).first() + from netbox_librenms_plugin.utils import find_by_librenms_id + + existing_device = find_by_librenms_id(Device, int(librenms_id), server_key) except (ValueError, TypeError): # librenms_id is not convertible to int; no match will be found existing_device = None @@ -264,45 +277,30 @@ def validate_device_for_import( result["existing_match_type"] = "librenms_id" result["can_import"] = False - # Check if name matches resolved name (accounts for use_sysname/strip_domain) - # Also accounts for virtual chassis naming pattern when device is a VC member - name_matched = False - if hostname and existing_device.name == hostname: - name_matched = True - elif ( - hostname - and hasattr(existing_device, "virtual_chassis") - and existing_device.virtual_chassis is not None - and existing_device.vc_position is not None - ): - # Device is a VC member — generate the expected VC name using - # the same function that the import creation process uses - expected_vc_name = _generate_vc_member_name( + # Detect legacy bare-integer format so UI can offer a migration action. + # Direct access needed to detect legacy integer format for migration prompt: + # LibreNMSAPI.get_librenms_id() returns an int in both formats, so only the + # raw type check on custom_field_data reveals whether migration is needed. + if isinstance(existing_device.custom_field_data.get("librenms_id"), int): + result["librenms_id_needs_migration"] = True + + # Check if name matches resolved name (VC-aware: compare against VC member name) + if hostname and existing_device.virtual_chassis and existing_device.vc_position: + vc_expected_name = _generate_vc_member_name( hostname, existing_device.vc_position, - serial=getattr(existing_device, "serial", None), + serial=existing_device.serial or "", ) - if existing_device.name == expected_vc_name: - name_matched = True - - if name_matched: + if existing_device.name == vc_expected_name: + result["name_matches"] = True + else: + result["name_sync_available"] = True + result["suggested_name"] = vc_expected_name + elif hostname and existing_device.name == hostname: result["name_matches"] = True - elif hostname: + elif hostname and existing_device.name != hostname: result["name_sync_available"] = True - # suggested_name uses resolved name (not raw sysName), - # respecting use_sysname/strip_domain preferences - if ( - hasattr(existing_device, "virtual_chassis") - and existing_device.virtual_chassis is not None - and existing_device.vc_position is not None - ): - result["suggested_name"] = _generate_vc_member_name( - hostname, - existing_device.vc_position, - serial=getattr(existing_device, "serial", None), - ) - else: - result["suggested_name"] = hostname + result["suggested_name"] = hostname # Check for serial drift on the linked device incoming_serial = libre_device.get("serial") or "" @@ -432,6 +430,10 @@ def validate_device_for_import( ) result["can_import"] = False + # Refresh local variable to reflect any VM-mode adjustments made during detection + # (e.g. existing VM found by hostname sets result["import_as_vm"] = True) + import_as_vm = result["import_as_vm"] + # Validate based on import type (Device or VM) if import_as_vm: # 2. For VMs: Validate Cluster (required) - Must be manually selected @@ -468,7 +470,10 @@ def validate_device_for_import( # 3. Validate DeviceType (required) hardware = libre_device.get("hardware", "") dt_match = match_librenms_hardware_to_device_type(hardware) - result["device_type"] = dt_match + # Update result keys individually to preserve the existing schema (especially "found") + result["device_type"]["found"] = dt_match["matched"] + result["device_type"]["device_type"] = dt_match.get("device_type") + result["device_type"]["match_type"] = dt_match.get("match_type") if not dt_match["matched"]: result["issues"].append(f"No matching device type found for hardware: '{hardware}'") @@ -482,11 +487,6 @@ def validate_device_for_import( } for dt in all_device_types ] - else: - # Rename 'matched' to 'found' for consistency - result["device_type"]["found"] = dt_match["matched"] - result["device_type"]["device_type"] = dt_match["device_type"] - result["device_type"]["match_type"] = dt_match["match_type"] # 4. DeviceRole (required) - Must be manually selected by user logger.debug(f"[{hostname}] Issues BEFORE adding role issue: {result['issues']}") @@ -511,9 +511,6 @@ def validate_device_for_import( available_racks = cache.get(cache_key) if available_racks is None: - from dcim.models import Rack - from django.db.models import Q - # Query racks for this site - include both: # 1. Racks assigned to locations within the site # 2. Racks directly assigned to the site (without location) @@ -559,20 +556,12 @@ def validate_device_for_import( ) if vc_detection: result["virtual_chassis"] = vc_detection - # Correct VC member suggested_names using the resolved name - # (which respects use_sysname/strip_domain preferences). - # This reuses the same function that BulkImportConfirmView uses. - if vc_detection.get("is_stack") and hostname: - update_vc_member_suggested_names(vc_detection, hostname) - logger.debug( - f"Virtual chassis CONFIRMED for device {hostname}: " - f"{vc_detection['member_count']} members" - ) - elif vc_detection["is_stack"]: + if vc_detection["is_stack"]: logger.debug( f"Virtual chassis CONFIRMED for device {hostname}: " f"{vc_detection['member_count']} members" ) + result["virtual_chassis"] = update_vc_member_suggested_names(vc_detection, hostname) except Exception as e: logger.exception(f"Exception during VC detection for device {hostname}: {e}") result["virtual_chassis"]["detection_error"] = str(e) @@ -694,6 +683,7 @@ def import_single_device( libre_device, use_sysname=use_sysname_opt, strip_domain=strip_domain_opt, + server_key=api.server_key, ) # Check if device already exists @@ -779,7 +769,7 @@ def import_single_device( "role": device_role, "status": "active" if libre_device.get("status") == 1 else "offline", "comments": f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - "custom_field_data": {"librenms_id": int(device_id)}, + "custom_field_data": {"librenms_id": {api.server_key: int(device_id)}}, } # Add optional fields diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py index 99aae78d5d..3e12658066 100644 --- a/netbox_librenms_plugin/import_utils/filters.py +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -1,5 +1,7 @@ -"""Device filtering and API queries for LibreNMS devices.""" +"""Device filtering and retrieval from LibreNMS.""" +import hashlib +import json import logging from typing import List @@ -33,9 +35,11 @@ def get_device_count_for_filters( """ devices = get_librenms_devices_for_import(api, filters=filters, force_refresh=clear_cache) - # Filter out disabled devices if requested + # Filter out disabled devices if requested. LibreNMS's "disabled" field (1=disabled, + # 0=enabled) reflects manual device disablement; "status" reflects SNMP reachability. + # show_disabled controls the former: hidden when disabled==1, shown regardless of status. if not show_disabled: - devices = [d for d in devices if d.get("status") == 1] + devices = [d for d in devices if int(d.get("disabled", 0)) != 1] return len(devices) @@ -85,10 +89,15 @@ def get_librenms_devices_for_import( if filters: # Check for status filter first - it has special handling if filters.get("status") is not None: + # Normalize to int: form fields send strings ("1"/"0"), API may send ints + try: + status_val = int(filters["status"]) + except (ValueError, TypeError): + status_val = None # Status filter uses special types that don't need query param - if filters["status"] == 1: + if status_val == 1: api_filters["type"] = "up" - elif filters["status"] == 0: + elif status_val == 0: api_filters["type"] = "down" # Save ALL other filters for client-side filtering when status is used @@ -170,8 +179,14 @@ def get_librenms_devices_for_import( # We'll filter client-side if needed # Use caching to avoid repeated API calls - # Include both API and client filters in cache key - cache_key = f"librenms_devices_import_{server_key}_{hash(str(api_filters))}_{hash(str(client_filters))}" + # Include both API and client filters in cache key (deterministic, cross-process stable). + # Use api.server_key (always resolved) rather than the raw server_key arg (may differ). + def _hash(d): + return hashlib.sha256( + json.dumps(sorted(d.items()) if isinstance(d, dict) else d, sort_keys=True).encode() + ).hexdigest()[:16] + + cache_key = f"librenms_devices_import_{api.server_key}_{_hash(api_filters)}_{_hash(client_filters)}" from_cache = False if force_refresh: diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py index e68e9b6680..b95f32eaec 100644 --- a/netbox_librenms_plugin/import_utils/virtual_chassis.py +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -1,4 +1,4 @@ -"""Virtual chassis detection, creation, and caching.""" +"""Virtual chassis detection, creation, and management.""" import logging from typing import List @@ -32,11 +32,11 @@ def _clone_virtual_chassis_data(data: dict | None) -> dict: members = [] for idx, member in enumerate(data.get("members", [])): member_copy = member.copy() - raw_position = member_copy.get("position", idx) + raw_position = member_copy.get("position", idx + 1) try: member_copy["position"] = int(raw_position) except (TypeError, ValueError): - member_copy["position"] = idx + member_copy["position"] = idx + 1 # 1-based fallback; position 0 is invalid members.append(member_copy) member_count = data.get("member_count") or len(members) @@ -175,7 +175,7 @@ def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> d logger.debug(f"VC detection: Found parent container at index {parent_index} for device {device_id}") break - if not parent_index: + if parent_index is None: return None # Step 3: Get children chassis at next level @@ -198,11 +198,13 @@ def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> d # Step 5: Extract member info members = [] for idx, chassis in enumerate(chassis_items): - raw_position = chassis.get("entPhysicalParentRelPos", idx) + # entPhysicalParentRelPos is 1-based; fall back to idx+1 (not idx) so + # position 0 is never produced — VC positions must be ≥ 1. + raw_position = chassis.get("entPhysicalParentRelPos", idx + 1) try: position = int(raw_position) except (TypeError, ValueError): - position = idx + position = idx + 1 member_data = { "serial": chassis.get("entPhysicalSerialNum", ""), "position": position, @@ -212,11 +214,12 @@ def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> d "description": chassis.get("entPhysicalDescr", ""), } - # Generate suggested name if we have master name + # Generate suggested name if we have master name. + # position is already 1-based, so pass it directly (no +1). if master_name: - member_data["suggested_name"] = _generate_vc_member_name(master_name, position + 1) + member_data["suggested_name"] = _generate_vc_member_name(master_name, position) else: - member_data["suggested_name"] = f"Member-{position + 1}" + member_data["suggested_name"] = f"Member-{position}" members.append(member_data) @@ -232,7 +235,19 @@ def detect_virtual_chassis_from_inventory(api: LibreNMSAPI, device_id: int) -> d return None -def _generate_vc_member_name(master_name: str, position: int, serial: str = None) -> str: +def _load_vc_member_name_pattern() -> str: + """Load the VC member name pattern from settings, with fallback to default.""" + from ..models import LibreNMSSettings + + try: + settings = LibreNMSSettings.objects.first() + return settings.vc_member_name_pattern if settings else "-M{position}" + except Exception as e: + logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") + return "-M{position}" + + +def _generate_vc_member_name(master_name: str, position: int, serial: str = None, pattern: str = None) -> str: """ Generate name for VC member device using configured pattern from settings. @@ -240,6 +255,9 @@ def _generate_vc_member_name(master_name: str, position: int, serial: str = None master_name: Name of the master/primary device position: VC position number serial: Optional serial number of the member device + pattern: Optional pre-loaded name pattern; if None, loaded from settings. + Pass a pre-loaded pattern when calling inside a loop to avoid + repeated DB queries. Returns: Generated member device name @@ -250,16 +268,8 @@ def _generate_vc_member_name(master_name: str, position: int, serial: str = None pattern="-SW{position}" -> "switch01-SW2" pattern=" [{serial}]" -> "switch01 [ABC123]" """ - # Import here to avoid circular dependency - from ..models import LibreNMSSettings - - # Get pattern from settings with fallback to default - try: - settings = LibreNMSSettings.objects.first() - pattern = settings.vc_member_name_pattern if settings else "-M{position}" - except Exception as e: - logger.warning(f"Could not load VC member name pattern from settings: {e}. Using default.") - pattern = "-M{position}" + if pattern is None: + pattern = _load_vc_member_name_pattern() # Prepare format variables format_vars = { @@ -294,6 +304,8 @@ def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: if not vc_data or not vc_data.get("is_stack"): return vc_data + # Load naming pattern once to avoid a DB query per member + vc_pattern = _load_vc_member_name_pattern() for idx, member in enumerate(vc_data.get("members", [])): raw_position = member.get("position", idx) try: @@ -302,7 +314,9 @@ def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: base_position = idx position = base_position + 1 # Convert to 1-based position member["position"] = base_position - member["suggested_name"] = _generate_vc_member_name(master_name, position, serial=member.get("serial")) + member["suggested_name"] = _generate_vc_member_name( + master_name, position, serial=member.get("serial"), pattern=vc_pattern + ) return vc_data @@ -334,10 +348,8 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis ] """ - # Store original master device state for rollback + # original_master_name is still referenced in warning messages inside the atomic block. original_master_name = master_device.name - original_vc = master_device.virtual_chassis - original_vc_position = master_device.vc_position try: with transaction.atomic(): @@ -360,7 +372,7 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis vc = VirtualChassis.objects.create( name=vc_name, master=master_device, - domain=f"librenms-{libre_device['device_id']}", + domain=f"librenms-{libre_device.get('device_id', master_device.pk)}", ) # Update master device @@ -371,10 +383,12 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis # Create member devices for remaining positions position = 2 # Start at 2 (master is 1) members_created = 0 + # Load naming pattern once to avoid a DB query per member + vc_pattern = _load_vc_member_name_pattern() for member in members_info: - # Skip if this is the master's serial - if member.get("serial") == master_device.serial: + # Skip if this is the master's serial (only when both serials are non-empty) + if member.get("serial") and member.get("serial") == master_device.serial: continue serial = member.get("serial") @@ -389,7 +403,24 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis logger.warning(f"Device with serial '{serial}' already exists, skipping VC member creation") continue - member_name = _generate_vc_member_name(master_base_name, position, serial=serial) + # Prefer the discovered SNMP position; fall back to sequential counter. + # Normalize discovered_pos: 0 is not a valid VC position, treat as absent. + try: + discovered_pos = int(member.get("position")) if member.get("position") is not None else None + except (TypeError, ValueError): + discovered_pos = None + if discovered_pos is not None and discovered_pos < 1: + discovered_pos = None # 0 is invalid for vc_position; fall back to counter + chosen_pos = discovered_pos if discovered_pos is not None else position + # Advance the sequential counter: + # - if discovered_pos was used, advance counter past it to avoid future reuse; + # - if counter was consumed as fallback, increment it normally. + if discovered_pos is None: + position += 1 + else: + position = max(position, discovered_pos + 1) + + member_name = _generate_vc_member_name(master_base_name, chosen_pos, serial=serial, pattern=vc_pattern) # Check for duplicate name if Device.objects.filter(name=member_name).exists(): @@ -406,15 +437,16 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis platform=master_device.platform, serial=serial, virtual_chassis=vc, - vc_position=position, + vc_position=chosen_pos, comments=f"VC member (LibreNMS: {member.get('name', 'Unknown')})\n" f"Auto-created from stack inventory", ) members_created += 1 - position += 1 # Validate member count - expected_members = len([m for m in members_info if m.get("serial") != master_device.serial]) + expected_members = len( + [m for m in members_info if not (m.get("serial") and m.get("serial") == master_device.serial)] + ) if members_created < expected_members: logger.warning( f"Created {members_created} members but expected {expected_members}. " @@ -429,12 +461,10 @@ def create_virtual_chassis_with_members(master_device: Device, members_info: lis return vc except Exception as e: - # Rollback master device to original state + # The transaction.atomic() block above will roll back all DB changes automatically. + # Manual state restoration is redundant and the save() would fail in a broken transaction. logger.error( - f"Virtual Chassis creation failed for device {master_device.name}: {e}. Rolling back master device changes." + f"Virtual Chassis creation failed for device {master_device.name}: {e}", + exc_info=True, ) - master_device.name = original_master_name - master_device.virtual_chassis = original_vc - master_device.vc_position = original_vc_position - master_device.save() raise diff --git a/netbox_librenms_plugin/import_utils/vm_operations.py b/netbox_librenms_plugin/import_utils/vm_operations.py index e7deb5d2fd..1272871a5a 100644 --- a/netbox_librenms_plugin/import_utils/vm_operations.py +++ b/netbox_librenms_plugin/import_utils/vm_operations.py @@ -1,4 +1,4 @@ -"""Virtual machine import operations.""" +"""Virtual machine creation and import operations.""" import logging @@ -13,7 +13,9 @@ logger = logging.getLogger(__name__) -def create_vm_from_librenms(libre_device: dict, validation: dict, use_sysname: bool = True, role=None): +def create_vm_from_librenms( + libre_device: dict, validation: dict, use_sysname: bool = True, role=None, server_key: str = "default" +): """ Create a NetBox VirtualMachine from LibreNMS device data. @@ -58,7 +60,7 @@ def create_vm_from_librenms(libre_device: dict, validation: dict, use_sysname: b role=role, # Optional VM role platform=platform, comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - custom_field_data={"librenms_id": int(libre_device["device_id"])}, + custom_field_data={"librenms_id": {server_key: int(libre_device["device_id"])}}, ) logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") @@ -162,6 +164,7 @@ def bulk_import_vms( api=api, use_sysname=use_sysname_opt, strip_domain=strip_domain_opt, + server_key=api.server_key, ) # Check if VM already exists @@ -206,7 +209,9 @@ def bulk_import_vms( libre_device["_computed_name"] = vm_name # Create VM - vm = create_vm_from_librenms(libre_device, validation, use_sysname=use_sysname, role=role) + vm = create_vm_from_librenms( + libre_device, validation, use_sysname=use_sysname, role=role, server_key=api.server_key + ) result["success"].append( { diff --git a/netbox_librenms_plugin/jobs.py b/netbox_librenms_plugin/jobs.py index 80c0903ed2..f56d564e46 100644 --- a/netbox_librenms_plugin/jobs.py +++ b/netbox_librenms_plugin/jobs.py @@ -113,6 +113,8 @@ def run( "filters": filters, "server_key": server_key, "vc_detection_enabled": vc_detection_enabled, + "use_sysname": use_sysname, + "strip_domain": strip_domain, "cache_timeout": api.cache_timeout, "cached_at": cached_at, "completed": True, diff --git a/netbox_librenms_plugin/librenms_api.py b/netbox_librenms_plugin/librenms_api.py index 5de9db6c25..8e2b2b3ac0 100644 --- a/netbox_librenms_plugin/librenms_api.py +++ b/netbox_librenms_plugin/librenms_api.py @@ -190,7 +190,9 @@ def get_librenms_id(self, obj): If found via API, stores ID in custom field if available, otherwise caches the value. """ - librenms_id = obj.cf.get("librenms_id") + from netbox_librenms_plugin.utils import get_librenms_device_id + + librenms_id = get_librenms_device_id(obj, self.server_key) if librenms_id: return librenms_id @@ -254,7 +256,9 @@ def _store_librenms_id(self, obj, librenms_id): None """ if "librenms_id" in obj.cf: - obj.custom_field_data["librenms_id"] = librenms_id + from netbox_librenms_plugin.utils import set_librenms_device_id + + set_librenms_device_id(obj, librenms_id, self.server_key) obj.save() else: # Use cache as fallback diff --git a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js index 08a0534714..2aeef777af 100644 --- a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js +++ b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js @@ -1100,6 +1100,12 @@ } showModal(modalElement, fallbackBackdropRef); + + // Re-initialize tooltips for newly swapped modal content + if (typeof bootstrap !== 'undefined' && bootstrap.Tooltip) { + const tooltipEls = modalContent.querySelectorAll('[data-bs-toggle="tooltip"]'); + [...tooltipEls].map(el => new bootstrap.Tooltip(el)); + } } document.body.addEventListener('htmx:afterSwap', ensureModalVisible); diff --git a/netbox_librenms_plugin/tables/device_status.py b/netbox_librenms_plugin/tables/device_status.py index e8b4fd8cf6..3f03764cef 100644 --- a/netbox_librenms_plugin/tables/device_status.py +++ b/netbox_librenms_plugin/tables/device_status.py @@ -479,15 +479,21 @@ def render_actions(self, value, record): btn_class = "btn-outline-warning" btn_icon = "mdi-information-outline" btn_label = " Details" + elif match_type == "librenms_id" and validation.get("librenms_id_needs_migration"): + btn_class = "btn-outline-warning" + btn_icon = "mdi-database-alert" + btn_label = " Legacy ID" else: btn_class = "btn-outline-success" btn_icon = "mdi-check-circle" btn_label = "" btn_title = "Resolve conflict" if (has_actions or has_mismatch) else "View details" + aria_attr = f'aria-label="{btn_title}" ' if btn_label == "" else "" buttons.append( f' @@ -129,9 +129,7 @@
New device {% endif %} - - {{ libre_device.sysName|default:libre_device.hostname }} - + {{ libre_device.sysName|default:libre_device.hostname }} {% if not validation.import_as_vm %} @@ -171,7 +169,7 @@
- @@ -184,7 +182,7 @@
{% csrf_token %} - @@ -192,8 +190,10 @@
{% endif %} {% elif validation.device_type.device_type %} - {{ validation.device_type.device_type }} - + + {{ validation.device_type.device_type }} + + {% else %} No matching type {% endif %} @@ -225,7 +225,7 @@
{% csrf_token %} - @@ -280,7 +280,7 @@
{% csrf_token %} - @@ -305,7 +305,7 @@
{% csrf_token %} - @@ -376,7 +376,13 @@
{% if validation.existing_device %} {% if validation.existing_match_type == 'librenms_id' %}
- Linked — ID {{ libre_device.device_id }} + {% if existing_id_servers %} + {% for srv in existing_id_servers %} + Linked — ID {{ srv.device_id }} @ {{ srv.display_name }} + {% endfor %} + {% else %} + Linked — ID {{ libre_device.device_id }} + {% endif %} {% if validation.name_matches %} Name match {% elif validation.name_sync_available %} @@ -392,7 +398,33 @@
{% if validation.device_type_mismatch %} Type mismatch {% endif %} + {% if validation.librenms_id_needs_migration %} + Legacy ID format + {% endif %}
+ {% if validation.librenms_id_needs_migration %} +
+
+ {% csrf_token %} + + + {% if not validation.serial_confirmed %} +
+ + +
+ {% endif %} + +
+
+ {% endif %} {% elif validation.existing_match_type == 'hostname' %}
@@ -409,7 +441,7 @@
{% endif %} — Exists as - {{ validation.existing_device.name }}, + {{ validation.existing_device.name }}, not linked to LibreNMS.
@@ -469,7 +501,7 @@
{% endif %} — Exists as - {{ validation.existing_device.name }}, + {{ validation.existing_device.name }}, not linked to LibreNMS. @@ -507,7 +539,7 @@
IP match — Device with IP {{ libre_device.ip }} exists as - {{ validation.existing_device.name }}. + {{ validation.existing_device.name }}. Consider adding LibreNMS ID manually. @@ -516,7 +548,7 @@
Exists — Device already exists as - {{ validation.existing_device.name }}. + {{ validation.existing_device.name }}.
{% endif %} @@ -559,17 +591,17 @@
{% if validation.existing_device %} {% if validation.import_as_vm or validation.existing_device.cluster %} + class="btn btn-primary btn-sm" target="_blank" rel="noopener noreferrer"> View VM in NetBox {% else %} + class="btn btn-primary btn-sm" target="_blank" rel="noopener noreferrer"> View in NetBox {% if validation.existing_match_type == 'librenms_id' %} + class="btn btn-outline-primary btn-sm" target="_blank" rel="noopener noreferrer"> Full Sync Page {% endif %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html index 82fd96ff03..64ec26cb63 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html @@ -29,7 +29,69 @@ {% block content %} -{% if librenms_server_info %} +{% if all_server_mappings %} +
+
+ LibreNMS Connections + {% if not librenms_server_info.is_legacy %} + + Change Server + + {% endif %} +
+
+ + + {% for mapping in all_server_mappings %} + + + + + + {% endfor %} + +
+ {% if mapping.is_active %} + + {% elif mapping.is_configured %} + + {% else %} + + {% endif %} + {% if mapping.is_configured %} + {{ mapping.display_name }} + {% else %} + {{ mapping.server_key }} + Not configured + {% endif %} + + {% if mapping.device_url %} + + ID {{ mapping.device_id }} + + + {% else %} + ID {{ mapping.device_id }} + {% endif %} + + {% if not mapping.is_configured %} +
+ {% csrf_token %} + + +
+ {% endif %} +
+
+
+{% elif librenms_server_info %}
diff --git a/netbox_librenms_plugin/tests/test_background_jobs.py b/netbox_librenms_plugin/tests/test_background_jobs.py index 13dca842d8..be7f7af377 100644 --- a/netbox_librenms_plugin/tests/test_background_jobs.py +++ b/netbox_librenms_plugin/tests/test_background_jobs.py @@ -686,6 +686,8 @@ def test_load_success_uses_correct_cache_keys(self, mock_job_class, mock_get_key "vc_detection_enabled": True, "cached_at": "2026-01-20T10:00:00Z", "cache_timeout": 600, + "use_sysname": True, + "strip_domain": False, } mock_job_class.objects.get.return_value = mock_job @@ -708,12 +710,16 @@ def test_load_success_uses_correct_cache_keys(self, mock_job_class, mock_get_key filters={"location": "dc1"}, device_id=1, vc_enabled=True, + use_sysname=True, + strip_domain=False, ) mock_get_key.assert_any_call( server_key="primary", filters={"location": "dc1"}, device_id=2, vc_enabled=True, + use_sysname=True, + strip_domain=False, ) assert len(results) == 2 @@ -734,6 +740,8 @@ def test_load_extracts_filters_from_job_data(self, mock_job_class, mock_get_key, "vc_detection_enabled": False, "cached_at": "2026-01-20T10:00:00Z", "cache_timeout": 300, + "use_sysname": True, + "strip_domain": False, } mock_job_class.objects.get.return_value = mock_job mock_get_key.return_value = "test_key" @@ -748,6 +756,8 @@ def test_load_extracts_filters_from_job_data(self, mock_job_class, mock_get_key, filters={"location": "dc2", "type": "router"}, device_id=1, vc_enabled=False, + use_sysname=True, + strip_domain=False, ) @patch("netbox_librenms_plugin.views.imports.list.cache") diff --git a/netbox_librenms_plugin/tests/test_import_utils.py b/netbox_librenms_plugin/tests/test_import_utils.py index 52c7ffd9af..b3c6096b9d 100644 --- a/netbox_librenms_plugin/tests/test_import_utils.py +++ b/netbox_librenms_plugin/tests/test_import_utils.py @@ -64,6 +64,31 @@ def test_get_import_device_cache_key(self): assert "secondary" in key assert "456" in key + def test_validated_device_cache_key_unique_per_naming_mode(self): + """Different naming preferences produce different cache keys.""" + from netbox_librenms_plugin.import_utils import get_validated_device_cache_key + + base_args = dict(server_key="default", filters={}, device_id=123, vc_enabled=False) + key_default = get_validated_device_cache_key(**base_args) + key_no_sysname = get_validated_device_cache_key(**base_args, use_sysname=False) + key_strip = get_validated_device_cache_key(**base_args, strip_domain=True) + + assert key_default != key_no_sysname + assert key_default != key_strip + assert key_no_sysname != key_strip + + def test_cache_metadata_key_unique_per_naming_mode(self): + """Different naming preferences produce different metadata cache keys.""" + from netbox_librenms_plugin.import_utils import get_cache_metadata_key + + base_args = dict(server_key="default", filters={}, vc_enabled=False) + key_default = get_cache_metadata_key(**base_args) + key_no_sysname = get_cache_metadata_key(**base_args, use_sysname=False) + key_strip = get_cache_metadata_key(**base_args, strip_domain=True) + + assert key_default != key_no_sysname + assert key_default != key_strip + # ============================================================================= # TestDeviceNameDetermination - 6 tests @@ -150,7 +175,7 @@ class TestDeviceRetrieval: """Test device retrieval and filtering functions.""" @patch("netbox_librenms_plugin.import_utils.filters.cache") - @patch("netbox_librenms_plugin.import_utils.device_operations.LibreNMSAPI") + @patch("netbox_librenms_plugin.import_utils.filters.LibreNMSAPI") def test_get_librenms_devices_for_import_success(self, mock_api_class, mock_cache): """Retrieve devices from LibreNMS API.""" mock_cache.get.return_value = None # Cache miss @@ -227,11 +252,11 @@ def test_get_device_count_for_filters_success(self, mock_cache): @patch("netbox_librenms_plugin.import_utils.filters.cache") def test_get_device_count_excludes_disabled(self, mock_cache): - """Count respects show_disabled filter parameter.""" + """Count respects show_disabled filter parameter: disabled==1 devices excluded.""" mock_cache.get.return_value = [ - {"device_id": 1, "hostname": "switch-01", "status": 1}, - {"device_id": 2, "hostname": "switch-02", "status": 1}, - {"device_id": 3, "hostname": "switch-03", "status": 0}, # disabled + {"device_id": 1, "hostname": "switch-01", "disabled": 0, "status": 1}, + {"device_id": 2, "hostname": "switch-02", "disabled": 0, "status": 0}, + {"device_id": 3, "hostname": "switch-03", "disabled": 1, "status": 1}, # disabled in LibreNMS ] mock_api = MagicMock() @@ -341,7 +366,6 @@ def test_validate_device_site_match_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [mock_site] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -399,7 +423,6 @@ def test_validate_device_site_not_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -461,7 +484,6 @@ def test_validate_device_platform_match_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -519,7 +541,6 @@ def test_validate_device_platform_not_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -577,7 +598,6 @@ def test_validate_device_type_match_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -638,7 +658,6 @@ def test_validate_device_type_not_found( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -651,7 +670,7 @@ def test_validate_device_type_not_found( result = validate_device_for_import(device_data, include_vc_detection=False) - assert result["device_type"]["matched"] is False + assert result["device_type"]["found"] is False assert any("device type" in issue.lower() for issue in result["issues"]) @patch("virtualization.models.VirtualMachine") @@ -698,7 +717,6 @@ def test_validate_device_role_required( } mock_role.objects.all.return_value = [MagicMock(id=1, name="Access Switch")] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [mock_site] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -757,7 +775,6 @@ def test_validate_device_handles_empty_location( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -816,7 +833,6 @@ def test_validate_device_handles_empty_os( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -877,7 +893,6 @@ def test_validate_device_handles_empty_hardware( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -891,7 +906,7 @@ def test_validate_device_handles_empty_hardware( result = validate_device_for_import(device_data, include_vc_detection=False) assert result is not None - assert result["device_type"]["matched"] is False + assert result["device_type"]["found"] is False @patch("virtualization.models.VirtualMachine") @patch("netbox_librenms_plugin.import_utils.device_operations.Device") @@ -974,7 +989,6 @@ def test_validate_device_returns_complete_state( } mock_role.objects.all.return_value = [] mock_cluster.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1000,7 +1014,6 @@ def test_validate_device_returns_complete_state( assert "platform" in result @patch("netbox_librenms_plugin.import_utils.device_operations.cache") - @patch("virtualization.models.Cluster") @patch("virtualization.models.VirtualMachine") @patch("netbox_librenms_plugin.import_utils.device_operations.Device") @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") @@ -1014,14 +1027,13 @@ def test_validate_device_import_as_vm( self, mock_site_model, mock_rack, - mock_cluster_module, + mock_cluster, mock_role, mock_match_type, mock_find_platform, mock_find_site, mock_device, mock_vm, - mock_cluster_local, mock_cache, ): """Import as VM mode uses cluster instead of site/device_type.""" @@ -1045,10 +1057,8 @@ def test_validate_device_import_as_vm( } mock_role.objects.all.return_value = [] mock_clusters = [MagicMock(id=1, name="VMware Cluster")] - # Cluster is imported at module level in device_operations - mock_cluster_module.objects.all.return_value = mock_clusters + mock_cluster.objects.all.return_value = mock_clusters mock_cache.get.return_value = None # Force cache miss to trigger Cluster.objects.all() - mock_rack.objects.filter.return_value = [] mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1104,201 +1114,15 @@ def test_validate_device_existing_vm_blocks_import( assert result["import_as_vm"] is True -class TestDeviceNamingPreferences: - """Test that validation honours use_sysname and strip_domain user preferences.""" - - COMMON_PATCHES = [ - "netbox_librenms_plugin.import_utils.device_operations.Site", - "netbox_librenms_plugin.import_utils.device_operations.Rack", - "netbox_librenms_plugin.import_utils.device_operations.Cluster", - "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", - "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", - "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", - "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", - "netbox_librenms_plugin.import_utils.device_operations.Device", - "virtualization.models.VirtualMachine", - ] - - def _setup_no_existing(self, mocks): - """Configure mocks so no existing device is found.""" - mock_vm = mocks[-1] # VirtualMachine - mock_device = mocks[-2] # Device - mock_find_site = mocks[-3] - mock_find_platform = mocks[-4] - mock_match_type = mocks[-5] - mock_role = mocks[-6] - mock_rack = mocks[-8] - mock_site_model = mocks[-9] - - mock_vm.objects.filter.return_value.first.return_value = None - mock_device.objects.filter.return_value.first.return_value = None - mock_find_site.return_value = { - "found": False, - "site": None, - "match_type": None, - "confidence": 0.0, - } - mock_find_platform.return_value = { - "found": False, - "platform": None, - "match_type": None, - } - mock_match_type.return_value = { - "matched": False, - "device_type": None, - "match_type": None, - } - mock_role.objects.all.return_value = [] - mock_rack.objects.filter.return_value = [] - mock_site_model.objects.all.return_value = [] - - @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.device_operations.Device") - @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") - @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") - @patch("netbox_librenms_plugin.import_utils.device_operations.Site") - def test_resolved_name_uses_sysname_by_default(self, *mocks): - """Default use_sysname=True uses sysName for resolved_name.""" - self._setup_no_existing(mocks) - from netbox_librenms_plugin.import_utils import validate_device_for_import - - device_data = { - "device_id": 1, - "hostname": "10.0.0.1", - "sysName": "core-switch", - } - result = validate_device_for_import(device_data, include_vc_detection=False) - assert result["resolved_name"] == "core-switch" - - @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.device_operations.Device") - @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") - @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") - @patch("netbox_librenms_plugin.import_utils.device_operations.Site") - def test_resolved_name_uses_hostname_when_sysname_disabled(self, *mocks): - """use_sysname=False uses hostname for resolved_name.""" - self._setup_no_existing(mocks) - from netbox_librenms_plugin.import_utils import validate_device_for_import - - device_data = { - "device_id": 1, - "hostname": "10.0.0.1", - "sysName": "core-switch", - } - result = validate_device_for_import( - device_data, - include_vc_detection=False, - use_sysname=False, - ) - assert result["resolved_name"] == "10.0.0.1" - - @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.device_operations.Device") - @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") - @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") - @patch("netbox_librenms_plugin.import_utils.device_operations.Site") - def test_resolved_name_strips_domain(self, *mocks): - """strip_domain=True strips the domain suffix.""" - self._setup_no_existing(mocks) - from netbox_librenms_plugin.import_utils import validate_device_for_import - - device_data = { - "device_id": 1, - "hostname": "switch-01.example.com", - "sysName": "switch-01.example.com", - } - result = validate_device_for_import( - device_data, - include_vc_detection=False, - strip_domain=True, - ) - assert result["resolved_name"] == "switch-01" - - @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.device_operations.Device") - @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") - @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") - @patch("netbox_librenms_plugin.import_utils.device_operations.Site") - def test_duplicate_detection_uses_resolved_name(self, *mocks): - """Duplicate detection should match against the resolved name, not raw hostname.""" - self._setup_no_existing(mocks) - - mock_device = mocks[-2] # Device - # The first filter call (librenms_id) returns None, - # the second filter call (name__iexact) returns the existing device. - existing = MagicMock() - existing.name = "core-switch" - existing.serial = "" - mock_device.objects.filter.return_value.first.side_effect = [None, existing] - - from netbox_librenms_plugin.import_utils import validate_device_for_import - - device_data = { - "device_id": 999, - "hostname": "10.0.0.1", - "sysName": "core-switch", - } - # use_sysname=True (default): resolved name is "core-switch" - # so duplicate detection should find existing device "core-switch" - result = validate_device_for_import(device_data, include_vc_detection=False) - - assert result["existing_device"] == existing - assert result["existing_match_type"] == "hostname" - - @patch("virtualization.models.VirtualMachine") - @patch("netbox_librenms_plugin.import_utils.device_operations.Device") - @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") - @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") - @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") - @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") - @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") - @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") - @patch("netbox_librenms_plugin.import_utils.device_operations.Site") - def test_backward_compatible_defaults(self, *mocks): - """Calling without naming params produces resolved_name in result.""" - self._setup_no_existing(mocks) - from netbox_librenms_plugin.import_utils import validate_device_for_import - - device_data = { - "device_id": 1, - "hostname": "switch-01", - } - result = validate_device_for_import(device_data, include_vc_detection=False) - - # resolved_name should be present and match sysName fallback to hostname - assert "resolved_name" in result - assert result["resolved_name"] == "switch-01" - - -class TestNameMatchesWithNamingPreferences: - """Test that name_matches/name_sync_available respect naming preferences and VC patterns. - - The name comparison should use the resolved name (result of _determine_device_name()) - which accounts for use_sysname and strip_domain, not the raw LibreNMS sysName. - For VC members, it should also account for the VC naming pattern. - """ +class TestSerialNumberMatching: + """Test serial number matching in device validation.""" - COMMON_PATCHES = [ + SERIAL_PATCHES = [ "netbox_librenms_plugin.import_utils.device_operations.Site", "netbox_librenms_plugin.import_utils.device_operations.Rack", "netbox_librenms_plugin.import_utils.device_operations.Cluster", "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", + "netbox_librenms_plugin.import_utils.device_operations.DeviceType", "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", @@ -1308,381 +1132,79 @@ class TestNameMatchesWithNamingPreferences: def _start_patches(self): """Start all common patches and return mocks in standard order.""" - self._patchers = [patch(p) for p in self.COMMON_PATCHES] + self._patchers = [patch(p) for p in self.SERIAL_PATCHES] mocks = [p.start() for p in self._patchers] ( self.mock_site_model, self.mock_rack, self.mock_cluster, self.mock_role, + self.mock_device_type, self.mock_match_type, self.mock_find_platform, self.mock_find_site, self.mock_device, self.mock_vm, ) = mocks + self.mock_device_type.objects.all.return_value = [] def _stop_patches(self): """Stop all patches.""" for p in self._patchers: p.stop() - def _configure_standard_mocks(self): - """Configure standard mock returns for site/platform/type/role.""" - self.mock_find_site.return_value = { - "found": True, - "site": MagicMock(), - "match_type": "exact", - "confidence": 1.0, - } - self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} - self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} - self.mock_role.objects.all.return_value = [] - self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] - self.mock_site_model.objects.all.return_value = [] - - def _setup_librenms_id_match(self, existing_device, as_vm=False): - """Configure mocks so that a device is found by librenms_id.""" - if as_vm: - self.mock_vm.objects.filter.return_value.first.return_value = existing_device - self.mock_device.objects.filter.return_value.first.return_value = None - else: - self.mock_vm.objects.filter.return_value.first.return_value = None - - def device_filter(**kwargs): - result = MagicMock() - if "custom_field_data__librenms_id" in kwargs: - result.first.return_value = existing_device - else: - result.first.return_value = None - return result - - self.mock_device.objects.filter.side_effect = device_filter - def setup_method(self): - """Set up common patches.""" + """Set up common patches for serial number tests.""" self._start_patches() def teardown_method(self): """Tear down patches.""" self._stop_patches() - def test_name_matches_with_strip_domain(self): - """strip_domain=True: FQDN in LibreNMS matches short name in NetBox.""" + def test_serial_match_blocks_import(self): + """Device with matching serial blocks import.""" existing = MagicMock() - existing.name = "router" - existing.serial = "" - existing.virtual_chassis = None - existing.vc_position = None + existing.name = "existing-device" + existing.serial = "ABC123" + + self.mock_vm.objects.filter.return_value.first.return_value = None - self._setup_librenms_id_match(existing) - self._configure_standard_mocks() + def device_filter(**kwargs): + result = MagicMock() + if "serial" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter from netbox_librenms_plugin.import_utils import validate_device_for_import - device_data = { - "device_id": 1, - "hostname": "router.example.com", - "sysName": "router.example.com", - } - result = validate_device_for_import( - device_data, - include_vc_detection=False, - strip_domain=True, - ) + device_data = {"device_id": 1, "hostname": "new-hostname", "serial": "ABC123"} + result = validate_device_for_import(device_data, include_vc_detection=False) - assert result["existing_match_type"] == "librenms_id" - assert result["name_matches"] is True - assert result["name_sync_available"] is False + assert result["can_import"] is False + assert result["existing_match_type"] == "serial" + assert result["existing_device"] == existing - def test_name_matches_uses_hostname_when_sysname_disabled(self): - """use_sysname=False: matches against hostname instead of sysName.""" + def test_serial_match_same_hostname_offers_link(self): + """Serial + hostname match offers link action.""" existing = MagicMock() - existing.name = "10.0.0.1" - existing.serial = "" - existing.virtual_chassis = None - existing.vc_position = None + existing.name = "switch-01" + existing.serial = "ABC123" - self._setup_librenms_id_match(existing) - self._configure_standard_mocks() - - from netbox_librenms_plugin.import_utils import validate_device_for_import - - device_data = { - "device_id": 1, - "hostname": "10.0.0.1", - "sysName": "core-switch", - } - result = validate_device_for_import( - device_data, - include_vc_detection=False, - use_sysname=False, - ) - - assert result["name_matches"] is True - assert result["name_sync_available"] is False - - def test_name_mismatch_offers_sync_with_resolved_name(self): - """When names don't match, suggested_name is the resolved name, not raw sysName.""" - existing = MagicMock() - existing.name = "old-device" - existing.serial = "" - existing.virtual_chassis = None - existing.vc_position = None - - self._setup_librenms_id_match(existing) - self._configure_standard_mocks() - - from netbox_librenms_plugin.import_utils import validate_device_for_import - - device_data = { - "device_id": 1, - "hostname": "new-switch.example.com", - "sysName": "new-switch.example.com", - } - result = validate_device_for_import( - device_data, - include_vc_detection=False, - strip_domain=True, - ) - - assert result["name_sync_available"] is True - # suggested_name should be the resolved (stripped) name, not raw sysName - assert result["suggested_name"] == "new-switch" - - @patch("netbox_librenms_plugin.import_utils.device_operations._generate_vc_member_name") - def test_name_matches_vc_member(self, mock_vc_name): - """VC member: name matches when existing device name matches generated VC name.""" - mock_vc_name.return_value = "switch-M2" - - existing = MagicMock() - existing.name = "switch-M2" - existing.serial = "SN123" - existing.virtual_chassis = MagicMock() # Not None → device is a VC member - existing.vc_position = 2 - - self._setup_librenms_id_match(existing) - self._configure_standard_mocks() - - from netbox_librenms_plugin.import_utils import validate_device_for_import - - device_data = { - "device_id": 1, - "hostname": "switch", - "sysName": "switch", - } - result = validate_device_for_import( - device_data, - include_vc_detection=False, - ) - - assert result["name_matches"] is True - assert result["name_sync_available"] is False - # _generate_vc_member_name should be called with resolved name, position, serial - mock_vc_name.assert_called_with("switch", 2, serial="SN123") - - @patch("netbox_librenms_plugin.import_utils.device_operations._generate_vc_member_name") - def test_name_matches_vc_member_with_strip_domain(self, mock_vc_name): - """VC member + strip_domain: FQDN resolved to short name matches VC pattern.""" - mock_vc_name.return_value = "siteA-9300-1 (2)" - - existing = MagicMock() - existing.name = "siteA-9300-1 (2)" - existing.serial = "SN456" - existing.virtual_chassis = MagicMock() - existing.vc_position = 2 - - self._setup_librenms_id_match(existing) - self._configure_standard_mocks() - - from netbox_librenms_plugin.import_utils import validate_device_for_import - - device_data = { - "device_id": 555, - "hostname": "siteA-9300-1.example.net.com", - "sysName": "siteA-9300-1.example.net.com", - } - result = validate_device_for_import( - device_data, - include_vc_detection=False, - strip_domain=True, - ) - - assert result["name_matches"] is True - assert result["name_sync_available"] is False - # Resolved name should be "siteA-9300-1" (stripped), then VC name generated - mock_vc_name.assert_called_with("siteA-9300-1", 2, serial="SN456") - - @patch("netbox_librenms_plugin.import_utils.device_operations._generate_vc_member_name") - def test_vc_member_name_mismatch_suggests_vc_name(self, mock_vc_name): - """VC member name mismatch: suggested_name should be the expected VC name.""" - mock_vc_name.return_value = "new-switch-M2" - - existing = MagicMock() - existing.name = "old-switch-M2" - existing.serial = "SN789" - existing.virtual_chassis = MagicMock() - existing.vc_position = 2 - - self._setup_librenms_id_match(existing) - self._configure_standard_mocks() - - from netbox_librenms_plugin.import_utils import validate_device_for_import - - device_data = { - "device_id": 1, - "hostname": "new-switch", - "sysName": "new-switch", - } - result = validate_device_for_import( - device_data, - include_vc_detection=False, - ) - - assert result["name_matches"] is False - assert result["name_sync_available"] is True - assert result["suggested_name"] == "new-switch-M2" - - def test_vm_name_matches_with_strip_domain(self): - """VM name comparison also uses resolved name, not raw sysName.""" - existing_vm = MagicMock() - existing_vm.name = "vm-server" - - self._setup_librenms_id_match(existing_vm, as_vm=True) - self._configure_standard_mocks() - - from netbox_librenms_plugin.import_utils import validate_device_for_import - - device_data = { - "device_id": 1, - "hostname": "vm-server.example.com", - "sysName": "vm-server.example.com", - } - result = validate_device_for_import( - device_data, - include_vc_detection=False, - strip_domain=True, - ) - - assert result["import_as_vm"] is True - assert result["name_matches"] is True - - def test_name_matches_exact_without_vc(self): - """Standalone device: exact name match works without VC check.""" - existing = MagicMock() - existing.name = "core-router" - existing.serial = "" - existing.virtual_chassis = None - existing.vc_position = None - - self._setup_librenms_id_match(existing) - self._configure_standard_mocks() - - from netbox_librenms_plugin.import_utils import validate_device_for_import - - device_data = { - "device_id": 1, - "hostname": "core-router", - "sysName": "core-router", - } - result = validate_device_for_import( - device_data, - include_vc_detection=False, - ) - - assert result["name_matches"] is True - assert result["name_sync_available"] is False - - -class TestSerialNumberMatching: - """Test serial number matching in device validation.""" - - SERIAL_PATCHES = [ - "netbox_librenms_plugin.import_utils.device_operations.Site", - "netbox_librenms_plugin.import_utils.device_operations.Rack", - "netbox_librenms_plugin.import_utils.device_operations.Cluster", - "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", - "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", - "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", - "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", - "netbox_librenms_plugin.import_utils.device_operations.Device", - "virtualization.models.VirtualMachine", - ] - - def _start_patches(self): - """Start all common patches and return mocks in standard order.""" - self._patchers = [patch(p) for p in self.SERIAL_PATCHES] - mocks = [p.start() for p in self._patchers] - ( - self.mock_site_model, - self.mock_rack, - self.mock_cluster, - self.mock_role, - self.mock_match_type, - self.mock_find_platform, - self.mock_find_site, - self.mock_device, - self.mock_vm, - ) = mocks - - def _stop_patches(self): - """Stop all patches.""" - for p in self._patchers: - p.stop() - - def setup_method(self): - """Set up common patches for serial number tests.""" - self._start_patches() - - def teardown_method(self): - """Tear down patches.""" - self._stop_patches() - - def test_serial_match_blocks_import(self): - """Device with matching serial blocks import.""" - existing = MagicMock() - existing.name = "existing-device" - existing.serial = "ABC123" - - self.mock_vm.objects.filter.return_value.first.return_value = None - - def device_filter(**kwargs): - result = MagicMock() - if "serial" in kwargs: - result.first.return_value = existing - else: - result.first.return_value = None - return result - - self.mock_device.objects.filter.side_effect = device_filter - - from netbox_librenms_plugin.import_utils import validate_device_for_import - - device_data = {"device_id": 1, "hostname": "new-hostname", "serial": "ABC123"} - result = validate_device_for_import(device_data, include_vc_detection=False) - - assert result["can_import"] is False - assert result["existing_match_type"] == "serial" - assert result["existing_device"] == existing - - def test_serial_match_same_hostname_offers_link(self): - """Serial + hostname match offers link action.""" - existing = MagicMock() - existing.name = "switch-01" - existing.serial = "ABC123" - - self.mock_vm.objects.filter.return_value.first.return_value = None - - def device_filter(**kwargs): - result = MagicMock() - if "serial" in kwargs: - result.first.return_value = existing - else: - result.first.return_value = None - return result - - self.mock_device.objects.filter.side_effect = device_filter + self.mock_vm.objects.filter.return_value.first.return_value = None + + def device_filter(**kwargs): + result = MagicMock() + if "serial" in kwargs: + result.first.return_value = existing + else: + result.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1718,7 +1240,7 @@ def device_filter(**kwargs): assert result["serial_action"] == "hostname_differs" assert result["existing_match_type"] == "serial" - assert "reinstalled" in result["warnings"][0] + assert "hostname differs" in result["warnings"][0] def test_hostname_match_diff_serial_offers_update(self): """Hostname matches but serial differs offers update_serial action.""" @@ -1759,7 +1281,6 @@ def _setup_no_match_mocks(self): self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] def test_serial_dash_ignored(self): @@ -1835,12 +1356,17 @@ def test_librenms_id_match_shows_serial_confirmed(self): existing = MagicMock() existing.name = "switch-01" existing.serial = "ABC123" + existing.virtual_chassis = None # Not a VC member → use plain hostname comparison + existing.vc_position = None self.mock_vm.objects.filter.return_value.first.return_value = None - def device_filter(**kwargs): + def device_filter(*args, **kwargs): result = MagicMock() - if "custom_field_data__librenms_id" in kwargs: + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + if q_has_librenms: result.first.return_value = existing else: result.first.return_value = None @@ -1857,7 +1383,6 @@ def device_filter(**kwargs): self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1875,12 +1400,17 @@ def test_librenms_id_match_detects_serial_drift(self): existing = MagicMock() existing.name = "switch-01" existing.serial = "OLD_SERIAL" + existing.virtual_chassis = None + existing.vc_position = None self.mock_vm.objects.filter.return_value.first.return_value = None - def device_filter(**kwargs): + def device_filter(*args, **kwargs): result = MagicMock() - if "custom_field_data__librenms_id" in kwargs: + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + if q_has_librenms: result.first.return_value = existing elif "serial" in kwargs: result.first.return_value = None @@ -1900,7 +1430,6 @@ def device_filter(**kwargs): self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1917,12 +1446,17 @@ def test_librenms_id_match_still_validates_site(self): existing = MagicMock() existing.name = "switch-01" existing.serial = "" + existing.virtual_chassis = None + existing.vc_position = None self.mock_vm.objects.filter.return_value.first.return_value = None - def device_filter(**kwargs): + def device_filter(*args, **kwargs): result = MagicMock() - if "custom_field_data__librenms_id" in kwargs: + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + if q_has_librenms: result.first.return_value = existing else: result.first.return_value = None @@ -1936,7 +1470,6 @@ def device_filter(**kwargs): self.mock_match_type.return_value = {"matched": True, "device_type": mock_dt, "match_type": "exact"} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] from netbox_librenms_plugin.import_utils import validate_device_for_import @@ -1957,6 +1490,8 @@ def test_existing_device_role_populated(self): existing = MagicMock() existing.name = "switch-01" existing.serial = "ABC123" + existing.virtual_chassis = None + existing.vc_position = None mock_existing_role = MagicMock() mock_existing_role.name = "Access Switch" existing.role = mock_existing_role @@ -1977,7 +1512,6 @@ def device_filter(**kwargs): self.mock_match_type.return_value = {"matched": True, "device_type": MagicMock(), "match_type": "exact"} self.mock_role.objects.all.return_value = [mock_existing_role] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: @@ -2003,6 +1537,8 @@ def test_device_type_mismatch_flagged(self): existing = MagicMock() existing.name = "switch-01" existing.serial = "ABC123" + existing.virtual_chassis = None + existing.vc_position = None existing_device_type = MagicMock() existing_device_type.pk = 1 existing_device_type.__str__ = lambda self: "Old Type" @@ -2033,7 +1569,6 @@ def device_filter(**kwargs): } self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: @@ -2058,6 +1593,8 @@ def test_no_device_type_mismatch_when_types_match(self): existing = MagicMock() existing.name = "switch-01" existing.serial = "ABC123" + existing.virtual_chassis = None + existing.vc_position = None same_device_type = MagicMock() same_device_type.pk = 1 existing.device_type = same_device_type @@ -2079,7 +1616,6 @@ def device_filter(**kwargs): self.mock_match_type.return_value = {"matched": True, "device_type": same_device_type, "match_type": "exact"} self.mock_role.objects.all.return_value = [] self.mock_cluster.objects.all.return_value = [] - self.mock_rack.objects.filter.return_value = [] self.mock_site_model.objects.all.return_value = [] with patch("netbox_librenms_plugin.import_utils.device_operations.cache") as mock_cache: @@ -2099,38 +1635,407 @@ def device_filter(**kwargs): assert result["device_type_mismatch"] is False -class TestDeviceConflictActionView: - """Test DeviceConflictActionView conflict resolution actions.""" - - def _create_view(self): - """Create a DeviceConflictActionView instance with mocked dependencies.""" - from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView - - view = object.__new__(DeviceConflictActionView) - view._librenms_api = MagicMock() - view._librenms_api.server_key = "default" - view.request = MagicMock() - view.request.user.has_perm.return_value = True - return view - - def _create_request(self, action, existing_device_id, use_sysname=False, strip_domain=False): - """Create a mock request with POST data.""" - request = MagicMock() - post_data = {"action": action, "existing_device_id": str(existing_device_id)} - if use_sysname: - post_data["use-sysname-toggle"] = "on" - if strip_domain: - post_data["strip-domain-toggle"] = "on" - request.POST = post_data - return request +class TestNameMatchesWithNamingPreferences: + """Test VC-aware name matching with use_sysname/strip_domain preferences.""" - @patch("netbox_librenms_plugin.views.imports.actions.cache") - @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") - def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache): - """Link action should set librenms_id and update name from sysName.""" - from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + PATCHES = [ + "netbox_librenms_plugin.import_utils.device_operations.Site", + "netbox_librenms_plugin.import_utils.device_operations.Rack", + "netbox_librenms_plugin.import_utils.device_operations.Cluster", + "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", + "netbox_librenms_plugin.import_utils.device_operations.DeviceType", + "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", + "netbox_librenms_plugin.import_utils.device_operations.Device", + "virtualization.models.VirtualMachine", + ] - view = self._create_view() + def setup_method(self): + self._patchers = [patch(p) for p in self.PATCHES] + mocks = [p.start() for p in self._patchers] + ( + self.mock_site, + self.mock_rack, + self.mock_cluster, + self.mock_role, + self.mock_device_type, + self.mock_match_type, + self.mock_find_platform, + self.mock_find_site, + self.mock_device, + self.mock_vm, + ) = mocks + self.mock_device_type.objects.all.return_value = [] + self.mock_vm.objects.filter.return_value.first.return_value = None + self.mock_find_site.return_value = { + "found": True, + "site": MagicMock(), + "match_type": "exact", + "confidence": 1.0, + } + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_site.objects.all.return_value = [] + + def teardown_method(self): + for p in self._patchers: + p.stop() + + def _make_existing(self, name, serial="SN001", virtual_chassis=None, vc_position=None): + existing = MagicMock() + existing.name = name + existing.serial = serial + existing.virtual_chassis = virtual_chassis + existing.vc_position = vc_position + existing.custom_field_data = {"librenms_id": {"default": 42}} + return existing + + def _setup_librenms_id_filter(self, existing): + def device_filter(*args, **kwargs): + result = MagicMock() + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + result.first.return_value = existing if q_has_librenms else None + result.exclude.return_value.first.return_value = None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + def test_strip_domain_name_matches(self): + """strip_domain=True resolves 'switch-01.example.com' to 'switch-01', matching existing device.""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + + existing = self._make_existing("switch-01") + self._setup_librenms_id_filter(existing) + + device_data = { + "device_id": 42, + "hostname": "switch-01.example.com", + "sysName": "switch-01.example.com", + "serial": "SN001", + } + result = validate_device_for_import(device_data, include_vc_detection=False, strip_domain=True) + assert result["name_matches"] is True + assert result["name_sync_available"] is False + + def test_sysname_disabled_uses_hostname(self): + """use_sysname=False falls back to hostname for name comparison.""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + + existing = self._make_existing("switch-hostname") + self._setup_librenms_id_filter(existing) + + device_data = { + "device_id": 42, + "hostname": "switch-hostname", + "sysName": "switch-sysname", + "serial": "SN001", + } + result = validate_device_for_import(device_data, include_vc_detection=False, use_sysname=False) + assert result["name_matches"] is True + assert result["resolved_name"] == "switch-hostname" + + def test_name_mismatch_offers_sync(self): + """When resolved name differs from existing device name, name_sync_available is set.""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + + existing = self._make_existing("old-name") + self._setup_librenms_id_filter(existing) + + device_data = { + "device_id": 42, + "hostname": "new-name", + "sysName": "new-name", + "serial": "SN001", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + assert result["name_matches"] is False + assert result["name_sync_available"] is True + assert result["suggested_name"] == "new-name" + + def test_vc_member_name_matches(self): + """Existing VC member name is compared against vc_member_name(hostname, vc_position).""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + from netbox_librenms_plugin.import_utils.virtual_chassis import _generate_vc_member_name + + mock_vc = MagicMock() + expected_name = _generate_vc_member_name("stack-master", 2, serial="SN001") + existing = self._make_existing(expected_name, serial="SN001", virtual_chassis=mock_vc, vc_position=2) + self._setup_librenms_id_filter(existing) + + device_data = { + "device_id": 42, + "hostname": "stack-master", + "sysName": "stack-master", + "serial": "SN001", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + assert result["name_matches"] is True + + def test_vc_member_name_mismatch_suggests_vc_name(self): + """When VC member name differs, suggested_name is the expected VC member name.""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + from netbox_librenms_plugin.import_utils.virtual_chassis import _generate_vc_member_name + + mock_vc = MagicMock() + existing = self._make_existing("wrong-name", serial="SN001", virtual_chassis=mock_vc, vc_position=2) + self._setup_librenms_id_filter(existing) + + device_data = { + "device_id": 42, + "hostname": "stack-master", + "sysName": "stack-master", + "serial": "SN001", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + expected_name = _generate_vc_member_name("stack-master", 2, serial="SN001") + assert result["name_matches"] is False + assert result["name_sync_available"] is True + assert result["suggested_name"] == expected_name + + def test_vc_member_with_strip_domain(self): + """strip_domain applies before VC member name comparison.""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + from netbox_librenms_plugin.import_utils.virtual_chassis import _generate_vc_member_name + + mock_vc = MagicMock() + expected_name = _generate_vc_member_name("stack", 1, serial="SN001") + existing = self._make_existing(expected_name, serial="SN001", virtual_chassis=mock_vc, vc_position=1) + self._setup_librenms_id_filter(existing) + + device_data = { + "device_id": 42, + "hostname": "stack.example.com", + "sysName": "stack.example.com", + "serial": "SN001", + } + result = validate_device_for_import(device_data, include_vc_detection=False, strip_domain=True) + assert result["name_matches"] is True + + def test_naming_criteria_populated(self): + """naming_criteria dict is set in result with use_sysname/strip_domain/source.""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + + self.mock_device.objects.filter.return_value.first.return_value = None + + device_data = { + "device_id": 99, + "hostname": "router-01", + "sysName": "router-sysname", + } + result = validate_device_for_import( + device_data, include_vc_detection=False, use_sysname=True, strip_domain=False + ) + criteria = result["naming_criteria"] + assert criteria is not None + assert criteria["use_sysname"] is True + assert criteria["strip_domain"] is False + assert criteria["raw_sysname"] == "router-sysname" + assert criteria["raw_hostname"] == "router-01" + assert criteria["source"] == "sysname" + + def test_naming_criteria_source_hostname_when_sysname_disabled(self): + """naming_criteria source is 'hostname' when use_sysname=False.""" + from netbox_librenms_plugin.import_utils import validate_device_for_import + + self.mock_device.objects.filter.return_value.first.return_value = None + + device_data = { + "device_id": 99, + "hostname": "router-01", + "sysName": "router-sysname", + } + result = validate_device_for_import( + device_data, include_vc_detection=False, use_sysname=False, strip_domain=False + ) + assert result["naming_criteria"]["source"] == "hostname" + + +class TestLegacyLibreNMSIdMigration: + """Test detection of legacy bare-integer librenms_id format during device validation.""" + + PATCHES = [ + "netbox_librenms_plugin.import_utils.device_operations.Site", + "netbox_librenms_plugin.import_utils.device_operations.Rack", + "netbox_librenms_plugin.import_utils.device_operations.Cluster", + "netbox_librenms_plugin.import_utils.device_operations.DeviceRole", + "netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_platform", + "netbox_librenms_plugin.import_utils.device_operations.find_matching_site", + "netbox_librenms_plugin.import_utils.device_operations.Device", + "virtualization.models.VirtualMachine", + ] + + def setup_method(self): + self._patchers = [patch(p) for p in self.PATCHES] + mocks = [p.start() for p in self._patchers] + ( + self.mock_site_model, + self.mock_rack, + self.mock_cluster, + self.mock_role, + self.mock_match_type, + self.mock_find_platform, + self.mock_find_site, + self.mock_device, + self.mock_vm, + ) = mocks + + self.mock_find_site.return_value = { + "found": True, + "site": MagicMock(), + "match_type": "exact", + "confidence": 1.0, + } + self.mock_find_platform.return_value = {"found": False, "platform": None, "match_type": None} + self.mock_match_type.return_value = {"matched": False, "device_type": None, "match_type": None} + self.mock_role.objects.all.return_value = [] + self.mock_cluster.objects.all.return_value = [] + self.mock_site_model.objects.all.return_value = [] + self.mock_vm.objects.filter.return_value.first.return_value = None + + def teardown_method(self): + for p in self._patchers: + p.stop() + + def _make_existing(self, librenms_id_value, serial="SN001"): + existing = MagicMock() + existing.name = "switch-01" + existing.serial = serial + existing.virtual_chassis = None + existing.vc_position = None + existing.custom_field_data = {"librenms_id": librenms_id_value} + return existing + + def _setup_device_filter(self, existing): + def device_filter(*args, **kwargs): + result = MagicMock() + q_has_librenms = any("librenms_id" in str(arg) for arg in args) or any( + k.startswith("custom_field_data__librenms_id") for k in kwargs + ) + result.first.return_value = existing if q_has_librenms else None + return result + + self.mock_device.objects.filter.side_effect = device_filter + + def test_legacy_int_sets_needs_migration_flag(self): + """Device with bare-integer librenms_id sets librenms_id_needs_migration=True.""" + existing = self._make_existing(librenms_id_value=42, serial="SN001") + self._setup_device_filter(existing) + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + result = validate_device_for_import( + {"device_id": 42, "hostname": "switch-01", "serial": "SN001"}, + include_vc_detection=False, + ) + + assert result["existing_match_type"] == "librenms_id" + assert result["librenms_id_needs_migration"] is True + assert result["serial_confirmed"] is True + + def test_legacy_int_no_serial_still_sets_flag(self): + """Legacy int format sets the migration flag even when serial is absent.""" + existing = self._make_existing(librenms_id_value=42, serial="") + self._setup_device_filter(existing) + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + result = validate_device_for_import( + {"device_id": 42, "hostname": "switch-01"}, + include_vc_detection=False, + ) + + assert result["librenms_id_needs_migration"] is True + assert result["serial_confirmed"] is False + + def test_json_format_does_not_set_flag(self): + """Device with JSON librenms_id does NOT set librenms_id_needs_migration.""" + existing = self._make_existing(librenms_id_value={"default": 42}, serial="SN001") + self._setup_device_filter(existing) + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + result = validate_device_for_import( + {"device_id": 42, "hostname": "switch-01", "serial": "SN001"}, + include_vc_detection=False, + ) + + assert result["existing_match_type"] == "librenms_id" + assert result["librenms_id_needs_migration"] is False + + def test_migrate_legacy_librenms_id_helper(self): + """migrate_legacy_librenms_id converts int to {server_key: int}.""" + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + obj = MagicMock() + obj.custom_field_data = {"librenms_id": 42} + result = migrate_legacy_librenms_id(obj, "primary") + + assert result is True + assert obj.custom_field_data["librenms_id"] == {"primary": 42} + + def test_migrate_legacy_librenms_id_noop_for_json(self): + """migrate_legacy_librenms_id is a no-op when value is already a dict.""" + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + obj = MagicMock() + obj.custom_field_data = {"librenms_id": {"primary": 42}} + result = migrate_legacy_librenms_id(obj, "primary") + + assert result is False + assert obj.custom_field_data["librenms_id"] == {"primary": 42} + + def test_migrate_legacy_librenms_id_noop_for_none(self): + """migrate_legacy_librenms_id is a no-op when librenms_id is absent.""" + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + obj = MagicMock() + obj.custom_field_data = {} + result = migrate_legacy_librenms_id(obj, "primary") + + assert result is False + + +class TestDeviceConflictActionView: + """Test DeviceConflictActionView conflict resolution actions.""" + + def _create_view(self): + """Create a DeviceConflictActionView instance with mocked dependencies.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = DeviceConflictActionView() + view._librenms_api = MagicMock() + view._librenms_api.server_key = "default" + view.request = MagicMock() + view.request.user.has_perm.return_value = True + return view + + def _create_request(self, action, existing_device_id, use_sysname=False, strip_domain=False): + """Create a mock request with POST data.""" + request = MagicMock() + # Always include both toggles so _resolve_naming_preferences never falls through + # to the user-pref/settings DB path, which would hit the real database. + post_data = { + "action": action, + "existing_device_id": str(existing_device_id), + "use-sysname-toggle": "on" if use_sysname else "off", + "strip-domain-toggle": "on" if strip_domain else "off", + } + request.POST = post_data + return request + + @patch("netbox_librenms_plugin.views.imports.actions.cache") + @patch("netbox_librenms_plugin.views.imports.actions.get_import_device_cache_key") + def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache): + """Link action should set librenms_id and update name from sysName.""" + from netbox_librenms_plugin.views.imports.actions import DeviceConflictActionView + + view = self._create_view() existing_device = MagicMock() existing_device.pk = 42 existing_device.custom_field_data = {} @@ -2151,15 +2056,21 @@ def test_link_action_sets_librenms_id_and_name(self, mock_cache_key, mock_cache) patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() view.post(request, device_id=10) - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} assert existing_device.name == "switch-01.example.com" existing_device.save.assert_called_once() @@ -2191,15 +2102,21 @@ def test_update_action_sets_hostname_serial_and_librenms_id(self, mock_cache_key patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() view.post(request, device_id=10) - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} assert existing_device.serial == "NEW-SERIAL" assert existing_device.name == "new-name.example.com" existing_device.save.assert_called_once() @@ -2227,15 +2144,21 @@ def test_update_serial_action_updates_serial_only(self, mock_cache_key, mock_cac patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() view.post(request, device_id=10) - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} assert existing_device.serial == "NEW-SERIAL" # Name should NOT be changed by update_serial assert existing_device.name == "switch-01" @@ -2264,8 +2187,13 @@ def test_update_skips_dash_serial(self, mock_cache_key, mock_cache): patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -2291,14 +2219,20 @@ def test_unknown_action_returns_400(self): request = self._create_request("invalid_action", 42) existing_device = MagicMock() + existing_device.pk = 42 libre_device = {"device_id": 10, "hostname": "switch-01", "serial": "ABC"} with ( patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, + patch.object(DeviceConflictActionView, "require_object_permissions", return_value=None), patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device - mock_validate.return_value = (libre_device, {}, {}) + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + # Include existing_device so the validated-conflict-target guard passes; + # we want to exercise the unknown-action branch, not the missing-device guard. + mock_validate.return_value = (libre_device, {"existing_device": existing_device}, {}) response = view.post(request, device_id=10) @@ -2333,6 +2267,8 @@ def test_sync_name_action_updates_name(self, mock_cache_key, mock_cache): patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -2363,6 +2299,8 @@ def test_device_type_mismatch_blocked_without_force(self, mock_cache_key, mock_c patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) response = view.post(request, device_id=10) @@ -2398,15 +2336,21 @@ def test_device_type_mismatch_allowed_with_force(self, mock_cache_key, mock_cach patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() view.post(request, device_id=10) - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} existing_device.save.assert_called_once() @patch("netbox_librenms_plugin.views.imports.actions.cache") @@ -2444,16 +2388,22 @@ def test_force_with_mismatch_updates_device_type(self, mock_cache_key, mock_cach patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() view.post(request, device_id=10) assert existing_device.device_type == librenms_device_type - assert existing_device.custom_field_data["librenms_id"] == 10 + assert existing_device.custom_field_data["librenms_id"] == {"default": 10} existing_device.save.assert_called_once() @patch("netbox_librenms_plugin.views.imports.actions.cache") @@ -2494,6 +2444,8 @@ def test_update_type_action_changes_device_type(self, mock_cache_key, mock_cache patch("dcim.models.Device") as mock_device_cls, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -2521,9 +2473,15 @@ def test_sync_serial_action(self, mock_cache_key, mock_cache): patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, + patch("netbox_librenms_plugin.views.imports.actions.transaction") as mock_tx, ): + mock_tx.atomic.return_value = MagicMock() mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.first.return_value = None mock_device_cls.objects.filter.return_value.exclude.return_value.first.return_value = None + mock_device_cls.objects.filter.return_value.exclude.return_value.exists.return_value = False mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -2552,10 +2510,14 @@ def test_sync_platform_action(self, mock_cache_key, mock_cache): patch.object(DeviceConflictActionView, "get_validated_device_with_selections") as mock_validate, patch.object(DeviceConflictActionView, "render_device_row") as mock_render, patch("dcim.models.Device") as mock_device_cls, - patch("dcim.models.Platform") as mock_platform_cls, + # Patch find_matching_platform at the utility module level — the action imports + # it from netbox_librenms_plugin.utils, so that is the correct seam to mock. + patch("netbox_librenms_plugin.utils.find_matching_platform") as mock_find_platform, ): mock_device_cls.objects.get.return_value = existing_device - mock_platform_cls.objects.get.return_value = mock_platform + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None + mock_find_platform.return_value = {"found": True, "platform": mock_platform, "match_type": "exact"} mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -2586,6 +2548,8 @@ def test_sync_device_type_action(self, mock_cache_key, mock_cache): patch("netbox_librenms_plugin.utils.match_librenms_hardware_to_device_type") as mock_hw_match, ): mock_device_cls.objects.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.get.return_value = existing_device + mock_device_cls.objects.select_for_update.return_value.filter.return_value.exclude.return_value.first.return_value = None mock_hw_match.return_value = {"matched": True, "device_type": new_device_type} mock_validate.return_value = (libre_device, validation, selections) mock_render.return_value = MagicMock() @@ -2672,6 +2636,50 @@ def test_platform_out_of_sync(self): assert result["platform_synced"] is False assert result["all_synced"] is False + def test_platform_no_match_found_returns_bool(self): + """When find_matching_platform returns no match, platform_synced must be False (not None).""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "ABC123" + existing.platform = MagicMock() # device has a platform set + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + libre_device = {"serial": "ABC123", "os": "ios", "hardware": "-"} + + with patch("netbox_librenms_plugin.utils.find_matching_platform") as mock_platform_match: + mock_platform_match.return_value = {"found": False, "platform": None} + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + # Without bool() cast this would be None; verify it's exactly False (type-stable) + assert result["platform_synced"] is False + assert isinstance(result["platform_synced"], bool) + + def test_platform_synced_no_netbox_platform_returns_bool(self): + """When device has no platform in NetBox and os is non-dash, platform_synced must be bool.""" + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + existing = MagicMock() + existing.serial = "ABC123" + existing.platform = None # no platform on device + device_type = MagicMock() + device_type.pk = 5 + existing.device_type = device_type + + libre_device = {"serial": "ABC123", "os": "eos", "hardware": "-"} + + with patch("netbox_librenms_plugin.utils.find_matching_platform") as mock_platform_match: + mock_platform_match.return_value = {"found": True, "platform": MagicMock()} + + result = DeviceValidationDetailsView._build_sync_info(libre_device, existing) + + # None and ... returns None; bool() cast ensures False + assert result["platform_synced"] is False + assert isinstance(result["platform_synced"], bool) + def test_hardware_no_match_device_type_out_of_sync(self): """When hardware is present but no device type match found, device_type_synced is False.""" from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView @@ -2692,3 +2700,855 @@ def test_hardware_no_match_device_type_out_of_sync(self): assert result["device_type_synced"] is False assert result["all_synced"] is False + + +class TestDeviceNamingPreferences: + """Test that validation honours use_sysname and strip_domain user preferences.""" + + def _setup_no_existing(self, mocks): + """Configure mocks so no existing device is found.""" + ( + mock_site_model, + mock_rack, + mock_cluster, + mock_role, + mock_match_type, + mock_find_platform, + mock_find_site, + mock_device, + mock_vm, + ) = mocks + + mock_vm.objects.filter.return_value.first.return_value = None + mock_device.objects.filter.return_value.first.return_value = None + mock_find_site.return_value = { + "found": False, + "site": None, + "match_type": None, + "confidence": 0.0, + } + mock_find_platform.return_value = { + "found": False, + "platform": None, + "match_type": None, + } + mock_match_type.return_value = { + "matched": False, + "device_type": None, + "match_type": None, + } + mock_role.objects.all.return_value = [] + mock_site_model.objects.all.return_value = [] + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_resolved_name_uses_sysname_by_default(self, *mocks): + """Default use_sysname=True uses sysName for resolved_name.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "10.0.0.1", + "sysName": "core-switch", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + assert result["resolved_name"] == "core-switch" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_resolved_name_uses_hostname_when_sysname_disabled(self, *mocks): + """use_sysname=False uses hostname for resolved_name.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "10.0.0.1", + "sysName": "core-switch", + } + result = validate_device_for_import( + device_data, + include_vc_detection=False, + use_sysname=False, + ) + assert result["resolved_name"] == "10.0.0.1" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_resolved_name_strips_domain(self, *mocks): + """strip_domain=True strips the domain suffix.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "switch-01.example.com", + "sysName": "switch-01.example.com", + } + result = validate_device_for_import( + device_data, + include_vc_detection=False, + strip_domain=True, + ) + assert result["resolved_name"] == "switch-01" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_duplicate_detection_uses_resolved_name(self, *mocks): + """Duplicate detection should match against the resolved name, not raw hostname.""" + self._setup_no_existing(mocks) + + # Unpack using same order as _setup_no_existing / @patch decorators (bottom-up) + ( + _mock_site, + _mock_rack, + _mock_cluster, + _mock_role, + _mock_hw, + _mock_platform, + _mock_find_site, + mock_device, + _mock_vm, + ) = mocks + existing = MagicMock() + existing.name = "core-switch" + existing.serial = "" + existing.virtual_chassis = None + existing.vc_position = None + mock_device.objects.filter.return_value.first.side_effect = [None, existing] + + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 999, + "hostname": "10.0.0.1", + "sysName": "core-switch", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + + assert result["existing_device"] == existing + assert result["existing_match_type"] == "hostname" + + @patch("virtualization.models.VirtualMachine") + @patch("netbox_librenms_plugin.import_utils.device_operations.Device") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_site") + @patch("netbox_librenms_plugin.import_utils.device_operations.find_matching_platform") + @patch("netbox_librenms_plugin.import_utils.device_operations.match_librenms_hardware_to_device_type") + @patch("netbox_librenms_plugin.import_utils.device_operations.DeviceRole") + @patch("netbox_librenms_plugin.import_utils.device_operations.Cluster") + @patch("netbox_librenms_plugin.import_utils.device_operations.Rack") + @patch("netbox_librenms_plugin.import_utils.device_operations.Site") + def test_backward_compatible_defaults(self, *mocks): + """Calling without naming params produces resolved_name in result.""" + self._setup_no_existing(mocks) + from netbox_librenms_plugin.import_utils import validate_device_for_import + + device_data = { + "device_id": 1, + "hostname": "switch-01", + } + result = validate_device_for_import(device_data, include_vc_detection=False) + assert "resolved_name" in result + assert result["resolved_name"] == "switch-01" + + +class TestProcessDeviceFilters: + """Tests for process_device_filters and related bulk_import utilities.""" + + def test_show_disabled_filters_integer_disabled_1(self): + """show_disabled=False should exclude devices with disabled==1 (int).""" + from unittest.mock import MagicMock, patch + + devices = [ + {"device_id": 1, "hostname": "a", "disabled": 0, "status": 1}, + {"device_id": 2, "hostname": "b", "disabled": 1, "status": 1}, # disabled in LibreNMS + ] + with ( + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_librenms_devices_for_import", + return_value=(devices, False), + ), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import", + side_effect=lambda d, **kw: { + "resolved_name": d["hostname"], + "is_ready": True, + "can_import": True, + "status": "active", + "existing_device": None, + "import_as_vm": False, + "existing_match_type": None, + }, + ), + patch("netbox_librenms_plugin.import_utils.bulk_import.prefetch_vc_data_for_devices"), + patch("netbox_librenms_plugin.import_utils.bulk_import.cache"), + patch("netbox_librenms_plugin.import_utils.bulk_import.get_cache_metadata_key", return_value="key"), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_validated_device_cache_key", return_value="vkey" + ), + ): + from netbox_librenms_plugin.import_utils.bulk_import import process_device_filters + + api = MagicMock() + api.server_key = "default" + result = process_device_filters( + api, filters={}, vc_detection_enabled=False, clear_cache=False, show_disabled=False + ) + + # Only enabled device (disabled==0) should be processed + assert len(result) == 1 + assert result[0]["hostname"] == "a" + + def test_show_disabled_keeps_unreachable_enabled_device(self): + """show_disabled=False should keep devices that are enabled (disabled==0) even if status==0.""" + from unittest.mock import MagicMock, patch + + devices = [ + {"device_id": 1, "hostname": "a", "disabled": 0, "status": 0}, # down but enabled + {"device_id": 2, "hostname": "b", "disabled": 1, "status": 0}, # down and disabled + ] + with ( + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_librenms_devices_for_import", + return_value=(devices, False), + ), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import", + side_effect=lambda d, **kw: { + "resolved_name": d["hostname"], + "is_ready": True, + "can_import": True, + "status": "active", + "existing_device": None, + "import_as_vm": False, + "existing_match_type": None, + }, + ), + patch("netbox_librenms_plugin.import_utils.bulk_import.prefetch_vc_data_for_devices"), + patch("netbox_librenms_plugin.import_utils.bulk_import.cache"), + patch("netbox_librenms_plugin.import_utils.bulk_import.get_cache_metadata_key", return_value="key"), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_validated_device_cache_key", return_value="vkey" + ), + ): + from netbox_librenms_plugin.import_utils.bulk_import import process_device_filters + + api = MagicMock() + api.server_key = "default" + result = process_device_filters( + api, filters={}, vc_detection_enabled=False, clear_cache=False, show_disabled=False + ) + + # Device a is enabled (disabled==0) and should be kept even though status==0 + assert len(result) == 1 + assert result[0]["hostname"] == "a" + + def test_show_disabled_true_includes_all(self): + """show_disabled=True should include both active and inactive devices.""" + from unittest.mock import MagicMock, patch + + devices = [ + {"device_id": 1, "hostname": "a", "status": 1}, + {"device_id": 2, "hostname": "b", "status": 0}, + ] + with ( + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_librenms_devices_for_import", + return_value=(devices, False), + ), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import", + side_effect=lambda d, **kw: { + "resolved_name": d["hostname"], + "is_ready": True, + "can_import": True, + "status": "active", + "existing_device": None, + "import_as_vm": False, + "existing_match_type": None, + }, + ), + patch("netbox_librenms_plugin.import_utils.bulk_import.prefetch_vc_data_for_devices"), + patch("netbox_librenms_plugin.import_utils.bulk_import.cache"), + patch("netbox_librenms_plugin.import_utils.bulk_import.get_cache_metadata_key", return_value="key"), + patch( + "netbox_librenms_plugin.import_utils.bulk_import.get_validated_device_cache_key", return_value="vkey" + ), + ): + from netbox_librenms_plugin.import_utils.bulk_import import process_device_filters + + api = MagicMock() + api.server_key = "default" + result = process_device_filters( + api, filters={}, vc_detection_enabled=False, clear_cache=False, show_disabled=True + ) + + assert len(result) == 2 + + def test_empty_return_helper(self): + """_empty_return should return ([], False) when return_cache_status=True, else [].""" + from netbox_librenms_plugin.import_utils.bulk_import import _empty_return + + assert _empty_return(True) == ([], False) + assert _empty_return(False) == [] + + def test_bulk_import_devices_uses_resolved_server_key(self): + """bulk_import_devices_shared should pass api.server_key to import_single_device.""" + from unittest.mock import MagicMock, patch + + with ( + patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") as mock_api_cls, + patch("netbox_librenms_plugin.import_utils.bulk_import.import_single_device") as mock_import, + patch("netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import"), + patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions"), + ): + mock_api = MagicMock() + mock_api.server_key = "resolved-key" + mock_api.get_device_info.return_value = (True, {"device_id": 1, "hostname": "sw"}) + mock_api_cls.return_value = mock_api + mock_import.return_value = {"success": True, "device": MagicMock(), "is_vm": False} + + user = MagicMock() + from netbox_librenms_plugin.import_utils.bulk_import import bulk_import_devices_shared + + bulk_import_devices_shared([1], user=user, server_key=None) + + # The resolved api.server_key ("resolved-key") must be passed, not None + assert mock_import.call_args is not None + assert mock_import.call_args.kwargs.get("server_key") == "resolved-key" + + +class TestVCPositionHandling: + """Test VC position normalization and suggested name generation.""" + + def test_clone_vc_data_position_fallback_is_one_based(self): + """_clone_virtual_chassis_data fallback must be 1-based (idx+1, not idx).""" + from netbox_librenms_plugin.import_utils.virtual_chassis import _clone_virtual_chassis_data + + data = {"is_stack": True, "member_count": 2, "members": [{"serial": "S1"}, {"serial": "S2"}]} + result = _clone_virtual_chassis_data(data) + positions = [m["position"] for m in result["members"]] + # First member: idx=0 → position should be 1, not 0 + assert positions[0] == 1 + assert positions[1] == 2 + + def test_clone_vc_data_preserves_explicit_positions(self): + """_clone_virtual_chassis_data must preserve explicitly set positions.""" + from netbox_librenms_plugin.import_utils.virtual_chassis import _clone_virtual_chassis_data + + data = { + "is_stack": True, + "member_count": 2, + "members": [{"serial": "S1", "position": 3}, {"serial": "S2", "position": 5}], + } + result = _clone_virtual_chassis_data(data) + assert result["members"][0]["position"] == 3 + assert result["members"][1]["position"] == 5 + + def test_clone_vc_data_bad_position_falls_back_to_one_based(self): + """_clone_virtual_chassis_data falls back to idx+1 for non-int position.""" + from netbox_librenms_plugin.import_utils.virtual_chassis import _clone_virtual_chassis_data + + data = { + "is_stack": True, + "member_count": 2, + "members": [{"serial": "S1", "position": "bad"}, {"serial": "S2", "position": None}], + } + result = _clone_virtual_chassis_data(data) + # idx=0 → fallback 1, idx=1 → fallback 2 + assert result["members"][0]["position"] == 1 + assert result["members"][1]["position"] == 2 + + def test_suggested_name_uses_position_directly(self): + """Suggested name generation must use position directly (not position+1). + + This test verifies that _generate_vc_member_name is called with the + already-1-based position value, not position+1. + """ + from netbox_librenms_plugin.import_utils.virtual_chassis import _generate_vc_member_name + + # position=1 should produce name with "1", not "2" + name = _generate_vc_member_name("switch-1", 1, pattern="-M{position}") + assert name == "switch-1-M1", f"Expected 'switch-1-M1', got '{name}'" + + # position=2 should produce "2", not "3" + name = _generate_vc_member_name("switch-1", 2, pattern="-M{position}") + assert name == "switch-1-M2", f"Expected 'switch-1-M2', got '{name}'" + + +class TestBulkImportCancellation: + """Test that bulk_import_devices_shared respects RQ and DB cancellation.""" + + def _run_bulk_import(self, mock_rq_job=None, db_status="running", device_ids=None): + """Helper: run bulk_import with provided mocks, return import call count.""" + from unittest.mock import MagicMock, patch + + if device_ids is None: + device_ids = [1, 2, 3, 4, 5, 6] + + job = MagicMock() + job.job.job_id = "test-uuid" + job_status = MagicMock() + job_status.value = db_status + job.job.status = job_status + job.logger = MagicMock() + + with ( + patch("netbox_librenms_plugin.import_utils.bulk_import.LibreNMSAPI") as mock_api_cls, + patch("netbox_librenms_plugin.import_utils.bulk_import.import_single_device") as mock_import, + patch("netbox_librenms_plugin.import_utils.bulk_import.validate_device_for_import"), + patch("netbox_librenms_plugin.import_utils.bulk_import.require_permissions"), + # Inline imports in the loop use django_rq.get_queue / rq.job.Job directly + patch("django_rq.get_queue") as mock_get_queue, + patch("rq.job.Job") as mock_rqjob_cls, + ): + mock_api = MagicMock() + mock_api.server_key = "default" + mock_api.get_device_info.return_value = (True, {"device_id": 1, "hostname": "sw"}) + mock_api_cls.return_value = mock_api + mock_import.return_value = {"success": True, "device": MagicMock(), "is_vm": False} + + if mock_rq_job is not None: + mock_conn = MagicMock() + mock_queue = MagicMock() + mock_queue.connection = mock_conn + mock_get_queue.return_value = mock_queue + mock_rqjob_cls.fetch.return_value = mock_rq_job + else: + # Simulate RQ unavailable — get_queue raises, triggers DB fallback + mock_get_queue.side_effect = Exception("RQ unavailable") + + from netbox_librenms_plugin.import_utils.bulk_import import bulk_import_devices_shared + + bulk_import_devices_shared(device_ids, user=MagicMock(), server_key=None, job=job) + + return mock_import.call_count + + def test_rq_stopped_cancels_import_loop(self): + """When RQ job is_stopped, import loop should break early.""" + rq_job = MagicMock() + rq_job.is_stopped = True + rq_job.is_failed = False + rq_job.get_status.return_value = "stopped" + + # With 6 devices and RQ stopped on first check (idx=1), at most 1 device processed + count = self._run_bulk_import(mock_rq_job=rq_job, device_ids=[1, 2, 3, 4, 5, 6]) + assert count == 0 # break before first import + + def test_rq_failed_cancels_import_loop(self): + """When RQ job is_failed, import loop should break early.""" + rq_job = MagicMock() + rq_job.is_stopped = False + rq_job.is_failed = True + rq_job.get_status.return_value = "failed" + + count = self._run_bulk_import(mock_rq_job=rq_job, device_ids=[1, 2, 3, 4, 5, 6]) + assert count == 0 + + def test_rq_unavailable_falls_back_to_db_check(self): + """When RQ is unavailable, DB status check is used as fallback.""" + # mock_rq_job=None triggers the side_effect=Exception path + count = self._run_bulk_import(mock_rq_job=None, db_status="failed", device_ids=[1]) + # With DB status "failed", import should not run + assert count == 0 + + def test_db_errored_status_also_terminates_loop(self): + """When DB job status is 'errored', import loop should terminate early.""" + count = self._run_bulk_import(mock_rq_job=None, db_status="errored", device_ids=[1, 2, 3]) + assert count == 0 + + def test_healthy_job_runs_all_devices(self): + """When job is healthy, all devices should be imported.""" + rq_job = MagicMock() + rq_job.is_stopped = False + rq_job.is_failed = False + rq_job.get_status.return_value = "started" + + count = self._run_bulk_import(mock_rq_job=rq_job, device_ids=[1, 2, 3]) + assert count == 3 + + +# --------------------------------------------------------------------------- +# Tests for DeviceValidationDetailsView._build_id_server_info +# --------------------------------------------------------------------------- + + +class TestBuildIdServerInfo: + """Test DeviceValidationDetailsView._build_id_server_info method.""" + + def _make_device(self, librenms_id_value): + from unittest.mock import MagicMock + + device = MagicMock() + device.custom_field_data = {"librenms_id": librenms_id_value} + return device + + def test_returns_none_for_legacy_int(self): + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + device = self._make_device(42) + result = DeviceValidationDetailsView._build_id_server_info(device) + assert result is None + + def test_returns_none_for_missing_cf(self): + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + device = self._make_device(None) + result = DeviceValidationDetailsView._build_id_server_info(device) + assert result is None + + def test_single_server_resolves_display_name(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + device = self._make_device({"production": 42}) + plugins_cfg = { + "netbox_librenms_plugin": { + "servers": { + "production": {"display_name": "Production LibreNMS", "librenms_url": "https://prod.example.com"}, + } + } + } + with patch("django.conf.settings") as mock_settings: + mock_settings.PLUGINS_CONFIG = plugins_cfg + result = DeviceValidationDetailsView._build_id_server_info(device) + + assert result is not None + assert len(result) == 1 + assert result[0]["server_key"] == "production" + assert result[0]["display_name"] == "Production LibreNMS" + assert result[0]["device_id"] == 42 + + def test_unconfigured_server_uses_key_as_display_name(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + device = self._make_device({"deleted-server": 77}) + plugins_cfg = {"netbox_librenms_plugin": {"servers": {}}} + with patch("django.conf.settings") as mock_settings: + mock_settings.PLUGINS_CONFIG = plugins_cfg + result = DeviceValidationDetailsView._build_id_server_info(device) + + assert result is not None + assert result[0]["display_name"] == "deleted-server" + + def test_empty_dict_returns_none(self): + from netbox_librenms_plugin.views.imports.actions import DeviceValidationDetailsView + + device = self._make_device({}) + result = DeviceValidationDetailsView._build_id_server_info(device) + assert result is None + + +# --------------------------------------------------------------------------- +# Tests for _refresh_existing_device sys_name fallback fix +# --------------------------------------------------------------------------- + + +class TestRefreshExistingDeviceSysNameFallback: + """Test that _refresh_existing_device tries sys_name even when hostname is empty.""" + + def test_sysname_used_when_hostname_empty(self): + """When hostname is empty but sys_name matches, the device is found in validation.""" + from unittest.mock import MagicMock, patch + + from netbox_librenms_plugin.import_utils.bulk_import import _refresh_existing_device + + mock_device = MagicMock() + mock_device.pk = 99 + mock_device.name = "router-01" + mock_device.custom_field_data = {"librenms_id": None} + + libre_device = { + "device_id": 55, + "hostname": "", # empty hostname + "sysName": "router-01", + "serial": "SN-MATCH", + } + validation = { + "existing_device": None, + "existing_vm": None, + "import_as_vm": False, + "is_ready": False, + "can_import": False, + } + + # sys_name lookup: filter(name__iexact="router-01") returns mock_device + # hostname lookup: filter(name__iexact="") returns None + def make_qs(return_val): + qs = MagicMock() + qs.first.return_value = return_val + return qs + + with patch("netbox_librenms_plugin.import_utils.bulk_import.find_by_librenms_id", return_value=None): + import dcim.models as dcim_models + import virtualization.models as virt_models + + with ( + patch.object( + dcim_models.Device.objects, + "filter", + side_effect=lambda **kw: make_qs(mock_device if kw.get("name__iexact") == "router-01" else None), + ), + patch.object(virt_models.VirtualMachine.objects, "filter", return_value=make_qs(None)), + ): + _refresh_existing_device(validation, libre_device=libre_device, server_key="default") + + assert validation["existing_device"] is mock_device + + def test_hostname_lookup_succeeds_without_sysname(self): + """When hostname is non-empty and matches, validation is updated correctly.""" + from unittest.mock import MagicMock, patch + + from netbox_librenms_plugin.import_utils.bulk_import import _refresh_existing_device + + mock_device = MagicMock() + mock_device.pk = 10 + mock_device.name = "sw-01" + mock_device.custom_field_data = {"librenms_id": None} + + libre_device = { + "device_id": 10, + "hostname": "sw-01", + "sysName": "sw-01-sysname", + "serial": "", + } + validation = { + "existing_device": None, + "existing_vm": None, + "import_as_vm": False, + "is_ready": False, + "can_import": False, + } + + def make_qs(return_val): + qs = MagicMock() + qs.first.return_value = return_val + return qs + + with patch("netbox_librenms_plugin.import_utils.bulk_import.find_by_librenms_id", return_value=None): + import dcim.models as dcim_models + import virtualization.models as virt_models + + with ( + patch.object( + dcim_models.Device.objects, + "filter", + side_effect=lambda **kw: make_qs(mock_device if kw.get("name__iexact") == "sw-01" else None), + ), + patch.object(virt_models.VirtualMachine.objects, "filter", return_value=make_qs(None)), + ): + _refresh_existing_device(validation, libre_device=libre_device, server_key="default") + + assert validation["existing_device"] is mock_device + + +# --------------------------------------------------------------------------- +# Tests for _get_hostname_for_action helper +# --------------------------------------------------------------------------- + + +class TestGetHostnameForAction: + """Test _get_hostname_for_action helper in actions.py.""" + + def test_returns_resolved_name_when_set(self): + from unittest.mock import MagicMock + + from netbox_librenms_plugin.views.imports.actions import _get_hostname_for_action + + request = MagicMock() + validation = {"resolved_name": "cached-name"} + libre_device = {"hostname": "raw-hostname", "sysName": "raw-sysname"} + + result = _get_hostname_for_action(request, validation, libre_device) + assert result == "cached-name" + + def test_falls_back_to_determine_device_name(self): + from unittest.mock import MagicMock, patch + + from netbox_librenms_plugin.views.imports.actions import _get_hostname_for_action + + request = MagicMock() + validation = {} # no resolved_name + libre_device = {"hostname": "host.example.com", "sysName": "host"} + + with patch("netbox_librenms_plugin.views.imports.actions._resolve_naming_preferences") as mock_prefs: + mock_prefs.return_value = (False, False) # use_sysname=False, strip_domain=False + with patch("netbox_librenms_plugin.views.imports.actions._determine_device_name") as mock_name: + mock_name.return_value = "host.example.com" + result = _get_hostname_for_action(request, validation, libre_device) + + assert result == "host.example.com" + mock_prefs.assert_called_once_with(request) + mock_name.assert_called_once() + + +# --------------------------------------------------------------------------- +# Tests for _resolve_naming_preferences underscore-variant key support +# --------------------------------------------------------------------------- + + +class TestResolveNamingPreferencesKeys: + """Test that _resolve_naming_preferences handles both hyphenated and underscored keys.""" + + def _make_request(self, post=None, get=None): + from unittest.mock import MagicMock + + request = MagicMock() + request.POST = post or {} + request.GET = get or {} + request.user = MagicMock() + return request + + def test_hyphenated_post_key_use_sysname(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import _resolve_naming_preferences + + request = self._make_request(post={"use-sysname-toggle": "on", "strip-domain-toggle": "off"}) + with patch("netbox_librenms_plugin.views.imports.actions.get_user_pref", return_value=None): + use_sysname, strip_domain = _resolve_naming_preferences(request) + assert use_sysname is True + assert strip_domain is False + + def test_underscored_post_key_use_sysname(self): + """Underscore variant 'use_sysname-toggle' should also be recognised.""" + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import _resolve_naming_preferences + + request = self._make_request(post={"use_sysname-toggle": "on", "strip_domain-toggle": "on"}) + with patch("netbox_librenms_plugin.views.imports.actions.get_user_pref", return_value=None): + use_sysname, strip_domain = _resolve_naming_preferences(request) + assert use_sysname is True + assert strip_domain is True + + def test_get_key_used_when_not_in_post(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import _resolve_naming_preferences + + request = self._make_request(get={"use-sysname-toggle": "off", "strip-domain-toggle": "on"}) + with patch("netbox_librenms_plugin.views.imports.actions.get_user_pref", return_value=None): + use_sysname, strip_domain = _resolve_naming_preferences(request) + assert use_sysname is False + assert strip_domain is True + + def test_user_pref_used_when_no_toggle_in_request(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import _resolve_naming_preferences + + request = self._make_request() + with patch("netbox_librenms_plugin.views.imports.actions.get_user_pref") as mock_pref: + mock_pref.side_effect = lambda req, key: False if "use_sysname" in key else True + use_sysname, strip_domain = _resolve_naming_preferences(request) + assert use_sysname is False + assert strip_domain is True + + def test_post_takes_precedence_over_user_pref(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import _resolve_naming_preferences + + request = self._make_request(post={"use-sysname-toggle": "off"}) + # user_pref would say True — POST should win + with patch("netbox_librenms_plugin.views.imports.actions.get_user_pref", return_value=True): + use_sysname, _ = _resolve_naming_preferences(request) + assert use_sysname is False + + def test_truthy_string_true_value(self): + """'true' and '1' (in addition to 'on') should be treated as True.""" + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import _resolve_naming_preferences + + for truthy_val in ("true", "True", "TRUE", "1"): + request = self._make_request(post={"use-sysname-toggle": truthy_val, "strip-domain-toggle": "off"}) + with patch("netbox_librenms_plugin.views.imports.actions.get_user_pref", return_value=None): + use_sysname, _ = _resolve_naming_preferences(request) + assert use_sysname is True, f"Expected True for value {truthy_val!r}" + + def test_falsy_string_false_value(self): + """Unrecognised strings should be treated as False.""" + from unittest.mock import patch + + from netbox_librenms_plugin.views.imports.actions import _resolve_naming_preferences + + for falsy_val in ("off", "false", "0", "", "no"): + request = self._make_request(post={"use-sysname-toggle": falsy_val, "strip-domain-toggle": "off"}) + with patch("netbox_librenms_plugin.views.imports.actions.get_user_pref", return_value=None): + use_sysname, _ = _resolve_naming_preferences(request) + assert use_sysname is False, f"Expected False for value {falsy_val!r}" + + +# --------------------------------------------------------------------------- +# Tests for vc_domain stack dedup key fix +# --------------------------------------------------------------------------- + + +class TestVCDomainStackDedup: + """Test that bulk_import_devices_shared deduplicates VC creation by member serials.""" + + def test_vc_domain_uses_member_serials(self): + """vc_domain for two stack members with the same serials should be identical.""" + # The logic lives inline; test the produced key directly from vc_data + members = [ + {"serial": "SN100", "position": 1}, + {"serial": "SN200", "position": 2}, + ] + member_serials = sorted(m.get("serial") for m in members if m.get("serial")) + vc_domain = f"librenms-stack-{','.join(member_serials)}" + + # Same members from a different device's perspective should produce the same key + assert vc_domain == "librenms-stack-SN100,SN200" + + def test_vc_domain_fallback_to_device_id_when_no_serials(self): + """When no member serials are available, device_id is used as fallback.""" + members = [ + {"position": 1}, + {"position": 2}, + ] + member_serials = sorted(m.get("serial") for m in members if m.get("serial")) + device_id = 42 + vc_domain = f"librenms-stack-{','.join(member_serials)}" if member_serials else f"librenms-{device_id}" + assert vc_domain == "librenms-42" + + def test_different_stacks_produce_different_keys(self): + """Two stacks with different serials produce distinct dedup keys.""" + members_a = [{"serial": "SN-A1"}, {"serial": "SN-A2"}] + members_b = [{"serial": "SN-B1"}, {"serial": "SN-B2"}] + key_a = f"librenms-stack-{','.join(sorted(m['serial'] for m in members_a))}" + key_b = f"librenms-stack-{','.join(sorted(m['serial'] for m in members_b))}" + assert key_a != key_b diff --git a/netbox_librenms_plugin/tests/test_permissions.py b/netbox_librenms_plugin/tests/test_permissions.py index 50bc2b6d04..bb6ea62caf 100644 --- a/netbox_librenms_plugin/tests/test_permissions.py +++ b/netbox_librenms_plugin/tests/test_permissions.py @@ -951,3 +951,130 @@ def test_delete_interfaces_invalid_type_raises_404(self): view = DeleteNetBoxInterfacesView() with pytest.raises(Http404): view.get_required_permissions_for_object_type("invalid") + + +# --------------------------------------------------------------------------- +# Tests for RemoveServerMappingView error handling (device_fields.py) +# --------------------------------------------------------------------------- + + +class TestRemoveServerMappingViewErrorHandling: + """Test RemoveServerMappingView handles full_clean/save failures gracefully.""" + + def _make_view(self, server_key, post_extra=None): + """Return a (view, request) pair with permissions satisfied.""" + from unittest.mock import MagicMock + + from netbox_librenms_plugin.views.sync.device_fields import RemoveServerMappingView + + request = MagicMock() + request.POST = {"server_key": server_key, **(post_extra or {})} + request.user = MagicMock() + request.user.has_perm.return_value = True + + view = RemoveServerMappingView() + view.request = request # required by mixin's has_write_permission + return view, request + + def test_validation_error_returns_error_message(self): + """ValidationError from full_clean leads to error message, not 500.""" + from unittest.mock import MagicMock, patch + + from django.core.exceptions import ValidationError + + view, request = self._make_view(server_key="orphan-server") + + mock_device = MagicMock() + mock_device.custom_field_data = {"librenms_id": {"orphan-server": 99}} + + mock_locked = MagicMock() + mock_locked.custom_field_data = {"librenms_id": {"orphan-server": 99}} + mock_locked.full_clean.side_effect = ValidationError("CF validation failed") + + plugins_cfg = {"netbox_librenms_plugin": {"servers": {}}} # orphan-server NOT configured + + with ( + patch("netbox_librenms_plugin.views.sync.device_fields.get_object_or_404", return_value=mock_device), + patch("netbox_librenms_plugin.views.sync.device_fields.Device") as mock_Device_cls, + patch("django.conf.settings") as mock_settings, + patch("netbox_librenms_plugin.views.sync.device_fields.messages") as mock_messages, + patch("netbox_librenms_plugin.views.sync.device_fields.redirect"), + patch("netbox_librenms_plugin.views.sync.device_fields.transaction") as mock_tx, + ): + mock_settings.PLUGINS_CONFIG = plugins_cfg + mock_Device_cls.objects.select_for_update.return_value.get.return_value = mock_locked + + # Make transaction.atomic() a no-op context manager + mock_tx.atomic.return_value.__enter__ = lambda s: None + mock_tx.atomic.return_value.__exit__ = lambda s, *a: None + mock_tx.set_rollback = MagicMock() + + view.post(request, pk=1) + + mock_messages.error.assert_called_once() + error_args = mock_messages.error.call_args[0] + assert "Validation error" in str(error_args[1]) or "CF validation failed" in str(error_args[1]) + + def test_configured_server_refused(self): + """Configured server mapping cannot be removed — error message shown.""" + from unittest.mock import MagicMock, patch + + view, request = self._make_view(server_key="active-server") + mock_device = MagicMock() + mock_device.custom_field_data = {"librenms_id": {"active-server": 5}} + + plugins_cfg = {"netbox_librenms_plugin": {"servers": {"active-server": {"librenms_url": "http://x"}}}} + + with ( + patch("netbox_librenms_plugin.views.sync.device_fields.get_object_or_404", return_value=mock_device), + patch("django.conf.settings") as mock_settings, + patch("netbox_librenms_plugin.views.sync.device_fields.messages") as mock_messages, + patch("netbox_librenms_plugin.views.sync.device_fields.redirect"), + ): + mock_settings.PLUGINS_CONFIG = plugins_cfg + view.post(request, pk=1) + + mock_messages.error.assert_called_once() + assert "Cannot remove" in mock_messages.error.call_args[0][1] + + def test_successful_removal_mutates_and_saves(self): + """Successful removal deletes the key from custom_field_data and saves the device.""" + from unittest.mock import MagicMock, patch + + view, request = self._make_view(server_key="orphan-server") + + mock_device = MagicMock() + mock_device.custom_field_data = {"librenms_id": {"orphan-server": 42, "other-server": 7}} + + mock_locked = MagicMock() + mock_locked.custom_field_data = {"librenms_id": {"orphan-server": 42, "other-server": 7}} + mock_locked.full_clean = MagicMock() + mock_locked.save = MagicMock() + + plugins_cfg = {"netbox_librenms_plugin": {"servers": {}}} # orphan-server NOT configured + + with ( + patch("netbox_librenms_plugin.views.sync.device_fields.get_object_or_404", return_value=mock_device), + patch("netbox_librenms_plugin.views.sync.device_fields.Device") as mock_Device_cls, + patch("django.conf.settings") as mock_settings, + patch("netbox_librenms_plugin.views.sync.device_fields.messages") as mock_messages, + patch("netbox_librenms_plugin.views.sync.device_fields.redirect"), + patch("netbox_librenms_plugin.views.sync.device_fields.transaction") as mock_tx, + ): + mock_settings.PLUGINS_CONFIG = plugins_cfg + mock_Device_cls.objects.select_for_update.return_value.get.return_value = mock_locked + + mock_tx.atomic.return_value.__enter__ = lambda s: None + mock_tx.atomic.return_value.__exit__ = lambda s, *a: None + mock_tx.set_rollback = MagicMock() + + view.post(request, pk=1) + + # The "orphan-server" key should have been removed and the device saved. + # Assert the exact shape of custom_field_data so misspelled keys are caught. + assert mock_locked.custom_field_data == {"librenms_id": {"other-server": 7}} + remaining = mock_locked.custom_field_data["librenms_id"] + assert "orphan-server" not in remaining + assert remaining.get("other-server") == 7 # sibling key preserved + mock_locked.save.assert_called_once() + mock_messages.success.assert_called_once() diff --git a/netbox_librenms_plugin/tests/test_sync_view_mismatch.py b/netbox_librenms_plugin/tests/test_sync_view_mismatch.py index e59ab6909c..e9a04d2997 100644 --- a/netbox_librenms_plugin/tests/test_sync_view_mismatch.py +++ b/netbox_librenms_plugin/tests/test_sync_view_mismatch.py @@ -318,3 +318,112 @@ def test_vc_pattern_no_match_leaves_name(self, mock_settings_qs, mock_hw): result = view.get_librenms_device_info(obj) assert result["mismatched_device"] is True + + +# --------------------------------------------------------------------------- +# Tests for _build_all_server_mappings +# --------------------------------------------------------------------------- + + +class TestBuildAllServerMappings: + """Tests for BaseLibreNMSSyncView._build_all_server_mappings.""" + + def _make_obj(self, cf_librenms_id): + obj = MagicMock() + obj.custom_field_data = {"librenms_id": cf_librenms_id} + return obj + + def test_returns_none_for_legacy_int(self): + from netbox_librenms_plugin.views.base.librenms_sync_view import BaseLibreNMSSyncView + + obj = self._make_obj(42) + result = BaseLibreNMSSyncView._build_all_server_mappings(obj, "production") + assert result is None + + def test_returns_none_for_missing_cf(self): + from netbox_librenms_plugin.views.base.librenms_sync_view import BaseLibreNMSSyncView + + obj = self._make_obj(None) + result = BaseLibreNMSSyncView._build_all_server_mappings(obj, "production") + assert result is None + + def test_single_configured_server(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.base.librenms_sync_view import BaseLibreNMSSyncView + + obj = self._make_obj({"production": 42}) + plugins_cfg = { + "netbox_librenms_plugin": { + "servers": { + "production": { + "display_name": "Production LibreNMS", + "librenms_url": "https://librenms.example.com", + }, + } + } + } + with patch("netbox_librenms_plugin.views.base.librenms_sync_view.django_settings") as mock_settings: + mock_settings.PLUGINS_CONFIG = plugins_cfg + result = BaseLibreNMSSyncView._build_all_server_mappings(obj, "production") + + assert result is not None + assert len(result) == 1 + entry = result[0] + assert entry["server_key"] == "production" + assert entry["device_id"] == 42 + assert entry["display_name"] == "Production LibreNMS" + assert entry["is_configured"] is True + assert entry["is_active"] is True + assert entry["device_url"] == "https://librenms.example.com/device/device=42/" + + def test_orphaned_server_is_not_configured(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.base.librenms_sync_view import BaseLibreNMSSyncView + + obj = self._make_obj({"deleted-server": 77}) + plugins_cfg = {"netbox_librenms_plugin": {"servers": {}}} + with patch("netbox_librenms_plugin.views.base.librenms_sync_view.django_settings") as mock_settings: + mock_settings.PLUGINS_CONFIG = plugins_cfg + result = BaseLibreNMSSyncView._build_all_server_mappings(obj, "production") + + assert result is not None + assert len(result) == 1 + entry = result[0] + assert entry["server_key"] == "deleted-server" + assert entry["device_id"] == 77 + assert entry["is_configured"] is False + assert entry["is_active"] is False + assert entry["device_url"] is None + + def test_multiple_servers_sorted_active_first(self): + from unittest.mock import patch + + from netbox_librenms_plugin.views.base.librenms_sync_view import BaseLibreNMSSyncView + + obj = self._make_obj({"mock-dev": 99, "production": 42, "old-server": 11}) + plugins_cfg = { + "netbox_librenms_plugin": { + "servers": { + "production": {"display_name": "Production", "librenms_url": "https://prod.example.com"}, + "mock-dev": {"display_name": "Mock", "librenms_url": "http://mock.example.com"}, + } + } + } + with patch("netbox_librenms_plugin.views.base.librenms_sync_view.django_settings") as mock_settings: + mock_settings.PLUGINS_CONFIG = plugins_cfg + result = BaseLibreNMSSyncView._build_all_server_mappings(obj, "production") + + assert result is not None + assert len(result) == 3 + # Active (production) first + assert result[0]["server_key"] == "production" + assert result[0]["is_active"] is True + # Configured (mock-dev) second + assert result[1]["server_key"] == "mock-dev" + assert result[1]["is_configured"] is True + assert result[1]["is_active"] is False + # Orphaned last + assert result[2]["server_key"] == "old-server" + assert result[2]["is_configured"] is False diff --git a/netbox_librenms_plugin/urls.py b/netbox_librenms_plugin/urls.py index 9eafdb1dca..63b4e9b0db 100644 --- a/netbox_librenms_plugin/urls.py +++ b/netbox_librenms_plugin/urls.py @@ -30,6 +30,7 @@ InterfaceTypeMappingView, LibreNMSImportView, LibreNMSSettingsView, + RemoveServerMappingView, SaveUserPrefView, SingleCableVerifyView, SingleInterfaceVerifyView, @@ -221,6 +222,11 @@ AssignVCSerialView.as_view(), name="assign_vc_serial", ), + path( + "devices//remove-server-mapping/", + RemoveServerMappingView.as_view(), + name="remove_server_mapping", + ), path( "device-status/", DeviceStatusListView.as_view(), diff --git a/netbox_librenms_plugin/utils.py b/netbox_librenms_plugin/utils.py index 4a5bf113a4..a5c73dd807 100644 --- a/netbox_librenms_plugin/utils.py +++ b/netbox_librenms_plugin/utils.py @@ -1,13 +1,17 @@ +import logging import re from typing import Optional from dcim.models import Device from django.core.exceptions import ObjectDoesNotExist +from django.db.models import Q from django.http import HttpRequest from netbox.config import get_config from netbox.plugins import get_plugin_config from utilities.paginator import get_paginate_count as netbox_get_paginate_count +logger = logging.getLogger(__name__) + def convert_speed_to_kbps(speed_bps: int) -> int: """ @@ -447,3 +451,109 @@ def check_vlan_group_matches( netbox_gid = netbox_tagged_group_ids.get(vid) return netbox_gid == selected_group_id return True + + +def get_librenms_device_id(obj, server_key: str = "default"): + """ + Get the LibreNMS device/port ID for a specific server from the JSON custom field. + + Supports both the legacy integer format and the new multi-server JSON format:: + + Legacy: librenms_id = 42 → returns 42 for any server_key + New: librenms_id = {"primary": 42} → returns 42 only for server_key="primary" + + Args: + obj: NetBox object with a ``librenms_id`` custom field. + server_key: LibreNMS server key (from plugin ``servers`` config). + + Returns: + int or None + """ + cf_value = obj.cf.get("librenms_id") + if cf_value is None: + return None + if isinstance(cf_value, int): + return cf_value # backward compat: bare integer from pre-migration + if isinstance(cf_value, dict): + return cf_value.get(server_key) + return None + + +def set_librenms_device_id(obj, device_id, server_key: str = "default"): + """ + Set the LibreNMS device/port ID for a specific server on the JSON custom field. + + Migrates any legacy bare-integer value to the dict format on first write. + + Args: + obj: NetBox object with a ``librenms_id`` custom field. + device_id: LibreNMS device ID (integer). + server_key: LibreNMS server key (from plugin ``servers`` config). + """ + cf_value = obj.custom_field_data.get("librenms_id") or {} + if isinstance(cf_value, int): + cf_value = {"default": cf_value} # migrate legacy value on first write + elif not isinstance(cf_value, dict): + logger.warning( + "librenms_id custom field has unexpected type %s on %r; resetting to empty dict.", + type(cf_value).__name__, + obj, + ) + cf_value = {} + cf_value[server_key] = device_id + obj.custom_field_data["librenms_id"] = cf_value + + +def find_by_librenms_id(model, librenms_id, server_key: str = "default"): + """ + Return the first object of *model* whose ``librenms_id`` JSON field contains + *librenms_id* under *server_key*. + + Also matches legacy records that stored ``librenms_id`` as a bare integer + directly in ``custom_field_data``. + + Args: + model: A Django model class (Device, VirtualMachine, Interface, …). + librenms_id: The LibreNMS device/port ID to look up. + server_key: LibreNMS server key (from plugin ``servers`` config). + + Returns: + Model instance or None + """ + return model.objects.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}) + | Q(custom_field_data__librenms_id=librenms_id) + ).first() + + +def migrate_legacy_librenms_id(obj, server_key: str = "default") -> bool: + """ + Migrate a legacy bare-integer ``librenms_id`` custom field to the JSON dict format, + scoped to *server_key*. + + Only performs the migration when the current value is a bare integer, i.e. a record + created before the multi-server JSON refactor. The integer is assumed to belong to + the server identified by *server_key* (the caller must verify this, e.g. by confirming + that the LibreNMS device ID and serial number both match). + + Does **not** call ``obj.save()`` — the caller is responsible for persisting the change. + + Args: + obj: NetBox object with a ``librenms_id`` custom field. + server_key: LibreNMS server key the legacy integer should be scoped to. + + Returns: + True if the value was migrated, False if it was already in the correct format. + """ + cf_value = obj.custom_field_data.get("librenms_id") + if not isinstance(cf_value, int): + return False + obj.custom_field_data["librenms_id"] = {server_key: cf_value} + logger.info( + "Migrated legacy librenms_id %d → {%r: %d} on %r", + cf_value, + server_key, + cf_value, + obj, + ) + return True diff --git a/netbox_librenms_plugin/views/__init__.py b/netbox_librenms_plugin/views/__init__.py index df3beac79b..f9c3db7790 100644 --- a/netbox_librenms_plugin/views/__init__.py +++ b/netbox_librenms_plugin/views/__init__.py @@ -53,6 +53,7 @@ from .sync.device_fields import ( # noqa: F401 AssignVCSerialView, CreateAndAssignPlatformView, + RemoveServerMappingView, UpdateDeviceNameView, UpdateDevicePlatformView, UpdateDeviceSerialView, diff --git a/netbox_librenms_plugin/views/base/cables_view.py b/netbox_librenms_plugin/views/base/cables_view.py index c390cdd539..4cbcdb7a36 100644 --- a/netbox_librenms_plugin/views/base/cables_view.py +++ b/netbox_librenms_plugin/views/base/cables_view.py @@ -4,6 +4,7 @@ from django.contrib import messages from django.core.cache import cache from django.core.exceptions import MultipleObjectsReturned +from django.db.models import Q from django.http import JsonResponse from django.shortcuts import get_object_or_404, render from django.urls import reverse @@ -78,10 +79,14 @@ def get_links_data(self, obj): def get_device_by_id_or_name(self, remote_device_id, hostname): """Try to find device in NetBox first by librenms_id custom field, then by name""" + server_key = self.librenms_api.server_key # First try matching by LibreNMS ID if remote_device_id: try: - device = Device.objects.get(custom_field_data__librenms_id=remote_device_id) + device = Device.objects.get( + Q(**{f"custom_field_data__librenms_id__{server_key}": remote_device_id}) + | Q(custom_field_data__librenms_id=remote_device_id) + ) return device, True, None except Device.DoesNotExist: pass @@ -116,13 +121,17 @@ def enrich_local_port(self, link, obj): if local_port := link.get("local_port"): interface = None local_port_id = link.get("local_port_id") + server_key = self.librenms_api.server_key if hasattr(obj, "virtual_chassis") and obj.virtual_chassis: chassis_member = get_virtual_chassis_member(obj, local_port) # First try to find interface by librenms_id if local_port_id: - interface = chassis_member.interfaces.filter(custom_field_data__librenms_id=local_port_id).first() + interface = chassis_member.interfaces.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": local_port_id}) + | Q(custom_field_data__librenms_id=local_port_id) + ).first() # Only if librenms_id match fails, try matching by name if not interface: @@ -130,7 +139,10 @@ def enrich_local_port(self, link, obj): else: # First try to find interface by librenms_id if local_port_id: - interface = obj.interfaces.filter(custom_field_data__librenms_id=local_port_id).first() + interface = obj.interfaces.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": local_port_id}) + | Q(custom_field_data__librenms_id=local_port_id) + ).first() # Only if librenms_id match fails, try matching by name if not interface: @@ -145,6 +157,7 @@ def enrich_remote_port(self, link, device): if remote_port := link.get("remote_port"): netbox_remote_interface = None librenms_remote_port_id = link.get("remote_port_id") + server_key = self.librenms_api.server_key # Handle virtual chassis case if hasattr(device, "virtual_chassis") and device.virtual_chassis: @@ -154,7 +167,8 @@ def enrich_remote_port(self, link, device): # First try to find interface by librenms_id if librenms_remote_port_id: netbox_remote_interface = chassis_member.interfaces.filter( - custom_field_data__librenms_id=librenms_remote_port_id + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id}) + | Q(custom_field_data__librenms_id=librenms_remote_port_id) ).first() # If not found by librenms_id, fall back to name matching on the correct chassis member @@ -165,7 +179,8 @@ def enrich_remote_port(self, link, device): # First try to find interface by librenms_id if librenms_remote_port_id: netbox_remote_interface = device.interfaces.filter( - custom_field_data__librenms_id=librenms_remote_port_id + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_remote_port_id}) + | Q(custom_field_data__librenms_id=librenms_remote_port_id) ).first() # If not found by librenms_id, fall back to name matching @@ -369,8 +384,10 @@ def post(self, request): # First try to find interface by librenms_id interface = None if local_port_id := link_data.get("local_port_id"): + _sk = self.librenms_api.server_key interface = selected_device.interfaces.filter( - custom_field_data__librenms_id=local_port_id + Q(**{f"custom_field_data__librenms_id__{_sk}": local_port_id}) + | Q(custom_field_data__librenms_id=local_port_id) ).first() # If not found by librenms_id, try matching by name diff --git a/netbox_librenms_plugin/views/base/ip_addresses_view.py b/netbox_librenms_plugin/views/base/ip_addresses_view.py index 22f4b49742..1ca3f6e290 100644 --- a/netbox_librenms_plugin/views/base/ip_addresses_view.py +++ b/netbox_librenms_plugin/views/base/ip_addresses_view.py @@ -11,7 +11,7 @@ from virtualization.models import VirtualMachine from netbox_librenms_plugin.tables.ipaddresses import IPAddressTable -from netbox_librenms_plugin.utils import get_interface_name_field +from netbox_librenms_plugin.utils import get_interface_name_field, get_librenms_device_id from netbox_librenms_plugin.views.mixins import CacheMixin, LibreNMSAPIMixin, LibreNMSPermissionMixin @@ -104,10 +104,11 @@ def _prefetch_netbox_data(self, obj): all_interfaces = list(obj.interfaces.all()) # Create maps for efficient lookups + server_key = self.librenms_api.server_key interfaces_by_librenms_id = { - interface.custom_field_data.get("librenms_id"): interface + get_librenms_device_id(interface, server_key): interface for interface in all_interfaces - if interface.custom_field_data.get("librenms_id") + if get_librenms_device_id(interface, server_key) } interfaces_by_name = {interface.name: interface for interface in all_interfaces} diff --git a/netbox_librenms_plugin/views/base/librenms_sync_view.py b/netbox_librenms_plugin/views/base/librenms_sync_view.py index 1346c3cb13..196c744907 100644 --- a/netbox_librenms_plugin/views/base/librenms_sync_view.py +++ b/netbox_librenms_plugin/views/base/librenms_sync_view.py @@ -1,5 +1,6 @@ import re +from django.conf import settings as django_settings from django.shortcuts import get_object_or_404, render from netbox.views import generic @@ -114,11 +115,60 @@ def get_context_data(self, request, obj): "platform_info": platform_info, "vc_inventory_serials": librenms_info["librenms_device_details"].get("vc_inventory_serials", []), "manufacturers": manufacturers, + "all_server_mappings": self._build_all_server_mappings(obj, self.librenms_api.server_key), } ) return context + @staticmethod + def _build_all_server_mappings(obj, active_server_key): + """Build a list of all LibreNMS server mappings for the given device. + + Each entry describes one server<->ID mapping stored in the ``librenms_id`` + custom field: + + * ``server_key`` – the key as stored in the CF dict. + * ``display_name`` – human-readable name from PLUGINS_CONFIG, or the key. + * ``librenms_url`` – base URL of that server (``None`` when not configured). + * ``device_id`` – the integer device ID on that server. + * ``device_url`` – direct URL to the device page on that server (or ``None``). + * ``is_configured`` – True when the server key exists in current plugin config. + * ``is_active`` – True when this is the currently active server. + + Returns ``None`` for legacy bare-int format (no per-server info to show) + and ``None`` when the CF is absent/invalid. + """ + cf_value = obj.custom_field_data.get("librenms_id") + if not isinstance(cf_value, dict) or not cf_value: + return None + + plugins_cfg = django_settings.PLUGINS_CONFIG.get("netbox_librenms_plugin", {}) + servers_config = plugins_cfg.get("servers", {}) + + result = [] + for sk, did in cf_value.items(): + srv_cfg = servers_config.get(sk) + is_configured = srv_cfg is not None + librenms_url = srv_cfg.get("librenms_url") if srv_cfg else None + display_name = (srv_cfg.get("display_name") or sk) if srv_cfg else sk + device_url = f"{librenms_url}/device/device={did}/" if librenms_url else None + result.append( + { + "server_key": sk, + "display_name": display_name, + "librenms_url": librenms_url, + "device_id": did, + "device_url": device_url, + "is_configured": is_configured, + "is_active": sk == active_server_key, + } + ) + + # Sort: active first, then configured, then orphaned + result.sort(key=lambda e: 0 if e["is_active"] else (1 if e["is_configured"] else 2)) + return result or None + def get_librenms_device_info(self, obj): """Get the LibreNMS device information for the given object.""" found_in_librenms = False diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 70c9395c8f..75ae108be2 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -6,6 +6,7 @@ from django.contrib import messages from django.core.cache import cache from django.core.exceptions import PermissionDenied, ValidationError +from django.db import transaction from django.http import HttpResponse, JsonResponse from django.shortcuts import redirect, render from django.utils.html import escape @@ -30,7 +31,7 @@ fetch_model_by_id, ) from netbox_librenms_plugin.tables.device_status import DeviceImportTable -from netbox_librenms_plugin.utils import get_user_pref, save_user_pref +from netbox_librenms_plugin.utils import get_user_pref, save_user_pref, set_librenms_device_id from netbox_librenms_plugin.views.mixins import LibreNMSAPIMixin, LibreNMSPermissionMixin, NetBoxObjectPermissionMixin logger = logging.getLogger(__name__) @@ -60,11 +61,24 @@ def _resolve_naming_preferences(request) -> tuple[bool, bool]: settings = None - # Check POST first (form submissions), then GET (HTMX hx-include on hx-get) - if "use-sysname-toggle" in request.POST: - use_sysname = request.POST.get("use-sysname-toggle") == "on" - elif "use-sysname-toggle" in request.GET: - use_sysname = request.GET.get("use-sysname-toggle") == "on" + # Check POST first (form submissions), then GET (HTMX hx-include on hx-get). + # Support hyphenated ("use-sysname-toggle"), underscored ("use_sysname-toggle"), + # and plain canonical ("use_sysname") key variants for compatibility across + # different form/hidden-input implementations. + _USE_SYSNAME_KEYS = ("use-sysname-toggle", "use_sysname-toggle", "use_sysname") + _STRIP_DOMAIN_KEYS = ("strip-domain-toggle", "strip_domain-toggle", "strip_domain") + _TRUTHY = frozenset({"on", "true", "1"}) + + def _is_truthy(val): + return val.lower() in _TRUTHY if val is not None else False + + _use_sysname_post = next((request.POST.get(k) for k in _USE_SYSNAME_KEYS if k in request.POST), None) + _use_sysname_get = next((request.GET.get(k) for k in _USE_SYSNAME_KEYS if k in request.GET), None) + + if _use_sysname_post is not None: + use_sysname = _is_truthy(_use_sysname_post) + elif _use_sysname_get is not None: + use_sysname = _is_truthy(_use_sysname_get) else: pref = get_user_pref(request, "plugins.netbox_librenms_plugin.use_sysname") if pref is not None: @@ -73,10 +87,13 @@ def _resolve_naming_preferences(request) -> tuple[bool, bool]: settings = LibreNMSSettings.objects.first() use_sysname = getattr(settings, "use_sysname_default", True) if settings else True - if "strip-domain-toggle" in request.POST: - strip_domain = request.POST.get("strip-domain-toggle") == "on" - elif "strip-domain-toggle" in request.GET: - strip_domain = request.GET.get("strip-domain-toggle") == "on" + _strip_domain_post = next((request.POST.get(k) for k in _STRIP_DOMAIN_KEYS if k in request.POST), None) + _strip_domain_get = next((request.GET.get(k) for k in _STRIP_DOMAIN_KEYS if k in request.GET), None) + + if _strip_domain_post is not None: + strip_domain = _is_truthy(_strip_domain_post) + elif _strip_domain_get is not None: + strip_domain = _is_truthy(_strip_domain_get) else: pref = get_user_pref(request, "plugins.netbox_librenms_plugin.strip_domain") if pref is not None: @@ -89,6 +106,20 @@ def _resolve_naming_preferences(request) -> tuple[bool, bool]: return use_sysname, strip_domain +def _get_hostname_for_action(request, validation: dict, libre_device: dict) -> str: + """Return the resolved hostname to use when updating a device during a conflict action. + + Prefer the cached ``resolved_name`` from validation (already computed with the + user's naming prefs at validation time). Fall back to computing it fresh from + the current request's naming preferences. + """ + resolved = validation.get("resolved_name") + if resolved: + return resolved + use_sysname, strip_domain = _resolve_naming_preferences(request) + return _determine_device_name(libre_device, use_sysname=use_sysname, strip_domain=strip_domain) + + class DeviceImportHelperMixin: """Mixin providing common validation and rendering helpers for device import views.""" @@ -177,6 +208,7 @@ def get_validated_device_with_selections(self, device_id: int, request) -> tuple include_vc_detection=enable_vc, use_sysname=use_sysname, strip_domain=strip_domain, + server_key=self.librenms_api.server_key, ) validation["import_as_vm"] = is_vm @@ -323,6 +355,7 @@ def post(self, request): api=self.librenms_api, use_sysname=use_sysname, strip_domain=strip_domain, + server_key=self.librenms_api.server_key, ) # Mark validation with VC detection flag for proper URL generation in table @@ -330,7 +363,7 @@ def post(self, request): vc_requested = request.GET.get("enable_vc_detection") == "true" validation["_vc_detection_enabled"] = vc_requested - device_name = validation["resolved_name"] + device_name = validation.get("resolved_name") if validation.get("virtual_chassis", {}).get("is_stack") and device_name: validation["virtual_chassis"] = update_vc_member_suggested_names( @@ -690,6 +723,7 @@ def post(self, request): # noqa: PLR0912 - branching keeps responses explicit import_as_vm=is_vm, api=None, # No VC detection needed for already-imported devices include_vc_detection=False, + server_key=self.librenms_api.server_key, use_sysname=sync_options.get("use_sysname", True), strip_domain=sync_options.get("strip_domain", False), ) @@ -778,6 +812,7 @@ def get(self, request, device_id): existing = validation.get("existing_device") if existing: context["sync_info"] = self._build_sync_info(libre_device, existing) + context["existing_id_servers"] = self._build_id_server_info(existing) return render( request, @@ -812,7 +847,7 @@ def _build_sync_info(libre_device, existing_device): netbox_platform = platform_info["netbox_platform"] matching_platform = platform_info["matching_platform"] - platform_synced = librenms_os == "-" or ( + platform_synced = librenms_os == "-" or bool( netbox_platform and matching_platform and netbox_platform.pk == matching_platform.pk ) @@ -843,6 +878,29 @@ def _build_sync_info(libre_device, existing_device): "all_synced": all_synced, } + @staticmethod + def _build_id_server_info(existing_device): + """Return per-server ID mappings for the existing device's librenms_id custom field. + + Returns a list of dicts with server_key, display_name, and device_id — one entry + per server the device is linked to. Returns None when the format is legacy (bare int) + or when the field is absent/invalid. + """ + from django.conf import settings + + cf_value = existing_device.custom_field_data.get("librenms_id") + if not isinstance(cf_value, dict): + return None + + plugins_config = settings.PLUGINS_CONFIG.get("netbox_librenms_plugin", {}) + servers_config = plugins_config.get("servers", {}) + result = [] + for sk, did in cf_value.items(): + srv_cfg = servers_config.get(sk, {}) + display_name = srv_cfg.get("display_name") or sk + result.append({"server_key": sk, "display_name": display_name, "device_id": did}) + return result or None + class DeviceRoleUpdateView(LibreNMSPermissionMixin, LibreNMSAPIMixin, DeviceImportHelperMixin, View): """HTMX view to update a table row when a role is selected.""" @@ -938,108 +996,125 @@ def post(self, request, device_id): librenms_device_type = validation.get("device_type", {}).get("device_type") librenms_id = libre_device.get("device_id") - - # Check for LibreNMS ID collision before any linking action + try: + librenms_id = int(librenms_id) + except (TypeError, ValueError): + return HttpResponse("Invalid or missing LibreNMS device_id in payload", status=400) + + # Wrap the LibreNMS-ID collision check and subsequent write in a single + # transaction so the read-then-write is atomic for link/update/update_serial. + # NOTE: A fully race-free guarantee would require a DB-unique constraint on + # (server_key, librenms_id) — e.g., a dedicated DeviceLibreNMSIDMapping model. + # That is deferred to a future schema migration. Until then, we acquire a + # row-level lock on the target device before re-checking for conflicts, which + # serializes concurrent operations on the SAME device and greatly reduces the + # window for assigning the same ID to two DIFFERENT devices. if action in {"link", "update", "update_serial"}: - id_conflict = ( - Device.objects.filter(custom_field_data__librenms_id=int(librenms_id)) - .exclude(pk=existing_device.pk) - .first() - ) - if id_conflict: - return HttpResponse( - f"LibreNMS ID conflict: ID {escape(str(librenms_id))} is already assigned to device " - f"'{escape(id_conflict.name)}' (ID: {id_conflict.pk})", - status=409, - ) - - if action == "link": - # Link to LibreNMS and update name from LibreNMS data - resolved_name = validation.get("resolved_name") - hostname = ( - resolved_name - if resolved_name - else _determine_device_name( - libre_device, - use_sysname=request.POST.get("use-sysname-toggle") == "on", - strip_domain=request.POST.get("strip-domain-toggle") == "on", - ) - ) - existing_device.custom_field_data["librenms_id"] = int(librenms_id) - existing_device.name = hostname - if librenms_device_type: - existing_device.device_type = librenms_device_type - if err := _save_device(existing_device): - return err - logger.info(f"Linked device '{existing_device.name}' to LibreNMS ID {librenms_id}") - - elif action == "update": - # Update hostname, serial, and link to LibreNMS - resolved_name = validation.get("resolved_name") - incoming_serial = libre_device.get("serial") or "" - hostname = ( - resolved_name - if resolved_name - else _determine_device_name( - libre_device, - use_sysname=request.POST.get("use-sysname-toggle") == "on", - strip_domain=request.POST.get("strip-domain-toggle") == "on", - ) - ) - existing_device.custom_field_data["librenms_id"] = int(librenms_id) - if incoming_serial and incoming_serial != "-": - conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() - if conflict_device: + from django.db.models import Q + + with transaction.atomic(): + server_key = self.librenms_api.server_key + # Lock the target device row so concurrent requests for the same + # device are serialized. The conflict check below is still a + # best-effort guard for different devices; a DB unique constraint + # would be needed for full protection. + try: + existing_device = Device.objects.select_for_update().get(pk=existing_device.pk) + except Device.DoesNotExist: return HttpResponse( - f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " - f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", + "Device no longer exists; it may have been deleted concurrently.", status=409, ) - existing_device.serial = incoming_serial - existing_device.name = hostname - if librenms_device_type: - existing_device.device_type = librenms_device_type - if err := _save_device(existing_device): - return err - logger.info( - f"Updated device '{existing_device.name}': serial={incoming_serial}, " - f"linked to LibreNMS ID {librenms_id}" - ) - - elif action == "update_serial": - # Update only the serial and link to LibreNMS - incoming_serial = libre_device.get("serial") or "" - existing_device.custom_field_data["librenms_id"] = int(librenms_id) - if incoming_serial and incoming_serial != "-": - conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() - if conflict_device: + conflict_exists = ( + Device.objects.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": librenms_id}) + | Q(custom_field_data__librenms_id=librenms_id) + ) + .exclude(pk=existing_device.pk) + .exists() + ) + if conflict_exists: return HttpResponse( - f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " - f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", + f"LibreNMS ID conflict: ID {librenms_id} is already assigned to another device.", status=409, ) - existing_device.serial = incoming_serial - if librenms_device_type: - existing_device.device_type = librenms_device_type - if err := _save_device(existing_device): - return err - logger.info( - f"Updated serial on device '{existing_device.name}' to {incoming_serial}, " - f"linked to LibreNMS ID {librenms_id}" - ) + + if action == "link": + # Link to LibreNMS and update name from LibreNMS data + hostname = _get_hostname_for_action(request, validation, libre_device) + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) + existing_device.name = hostname + if librenms_device_type: + existing_device.device_type = librenms_device_type + if err := _save_device(existing_device): + return err + logger.info(f"Linked device '{existing_device.name}' to LibreNMS ID {librenms_id}") + + elif action == "update": + # Update hostname, serial, and link to LibreNMS + hostname = _get_hostname_for_action(request, validation, libre_device) + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-": + # Lock any conflicting device under the same transaction to reduce + # the serial-assignment race window (best-effort; a DB unique + # constraint on serial would give full protection). + conflict_device = ( + Device.objects.select_for_update() + .filter(serial=incoming_serial) + .exclude(pk=existing_device.pk) + .first() + ) + if conflict_device: + return HttpResponse( + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", + status=409, + ) + existing_device.serial = incoming_serial + existing_device.name = hostname + if librenms_device_type: + existing_device.device_type = librenms_device_type + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) + if err := _save_device(existing_device): + return err + logger.info( + f"Updated device '{existing_device.name}': serial={incoming_serial}, " + f"linked to LibreNMS ID {librenms_id}" + ) + + elif action == "update_serial": + # Update only the serial and link to LibreNMS + incoming_serial = libre_device.get("serial") or "" + if incoming_serial and incoming_serial != "-": + # Lock any conflicting device under the same transaction to reduce + # the serial-assignment race window (best-effort; a DB unique + # constraint on serial would give full protection). + conflict_device = ( + Device.objects.select_for_update() + .filter(serial=incoming_serial) + .exclude(pk=existing_device.pk) + .first() + ) + if conflict_device: + return HttpResponse( + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", + status=409, + ) + existing_device.serial = incoming_serial + if librenms_device_type: + existing_device.device_type = librenms_device_type + set_librenms_device_id(existing_device, librenms_id, self.librenms_api.server_key) + if err := _save_device(existing_device): + return err + logger.info( + f"Updated serial on device '{existing_device.name}' to {incoming_serial}, " + f"linked to LibreNMS ID {librenms_id}" + ) elif action == "sync_name": # Sync device name from LibreNMS (e.g., IP → sysName) - resolved_name = validation.get("resolved_name") - hostname = ( - resolved_name - if resolved_name - else _determine_device_name( - libre_device, - use_sysname=request.POST.get("use-sysname-toggle") == "on", - strip_domain=request.POST.get("strip-domain-toggle") == "on", - ) - ) + hostname = _get_hostname_for_action(request, validation, libre_device) existing_device.name = hostname if err := _save_device(existing_device): return err @@ -1056,25 +1131,35 @@ def post(self, request, device_id): return HttpResponse("No LibreNMS device type available to update", status=400) elif action == "sync_serial": - # Sync serial number from LibreNMS + # Sync serial number from LibreNMS. + # Wrap conflict-check-and-write in a transaction with a row lock so + # concurrent requests cannot both pass the serial uniqueness guard. incoming_serial = libre_device.get("serial") or "" if incoming_serial and incoming_serial != "-": - # Check for serial ownership conflict - conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=existing_device.pk).first() - if conflict_device: - logger.warning( - f"Serial sync blocked: '{incoming_serial}' already assigned to " - f"'{conflict_device.name}' (pk={conflict_device.pk})" - ) - return HttpResponse( - f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " - f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", - status=409, - ) - existing_device.serial = incoming_serial - if err := _save_device(existing_device): - return err - logger.info(f"Synced serial on '{existing_device.name}' to {incoming_serial}") + with transaction.atomic(): + try: + locked_device = Device.objects.select_for_update().get(pk=existing_device.pk) + except Device.DoesNotExist: + return HttpResponse( + "Device no longer exists; it may have been deleted concurrently.", + status=409, + ) + # Re-check for serial ownership conflict under lock + conflict_device = Device.objects.filter(serial=incoming_serial).exclude(pk=locked_device.pk).first() + if conflict_device: + logger.warning( + f"Serial sync blocked: '{incoming_serial}' already assigned to " + f"'{conflict_device.name}' (pk={conflict_device.pk})" + ) + return HttpResponse( + f"Serial conflict: '{escape(incoming_serial)}' is already assigned to device " + f"'{escape(conflict_device.name)}' (ID: {conflict_device.pk})", + status=409, + ) + locked_device.serial = incoming_serial + if err := _save_device(locked_device): + return err + logger.info(f"Synced serial on '{locked_device.name}' to {incoming_serial}") else: return HttpResponse("No valid serial from LibreNMS", status=400) @@ -1109,6 +1194,42 @@ def post(self, request, device_id): else: return HttpResponse(f"No matching device type for '{escape(hardware)}'", status=400) + elif action == "migrate_librenms_id": + # Migrate legacy bare-integer librenms_id to the JSON dict format. + # Only safe when the integer matches the LibreNMS device ID for this server, + # confirmed by serial match (or explicit force). + from netbox_librenms_plugin.utils import migrate_legacy_librenms_id + + # Direct access needed to detect legacy integer format for migration prompt: + # LibreNMSAPI.get_librenms_id() returns an int in both formats; only the raw + # type check on custom_field_data reveals whether migration is needed. + cf_value = existing_device.custom_field_data.get("librenms_id") + if not isinstance(cf_value, int): + return HttpResponse( + "Device librenms_id is already in JSON format; no migration needed.", + status=400, + ) + # Verify the stored legacy ID matches the active LibreNMS device_id so we don't + # migrate a stale/incorrect association to the wrong server mapping. + if cf_value != librenms_id: + return HttpResponse( + f"Legacy librenms_id ({cf_value}) does not match the active device ID " + f"({librenms_id}); cannot migrate safely.", + status=400, + ) + if not validation.get("serial_confirmed") and not force: + return HttpResponse( + "Serial number not confirmed. Check the force checkbox to migrate without serial verification.", + status=400, + ) + migrate_legacy_librenms_id(existing_device, self.librenms_api.server_key) + if err := _save_device(existing_device): + return err + logger.info( + f"Migrated legacy librenms_id on '{existing_device.name}' " + f"to {{{self.librenms_api.server_key!r}: {cf_value}}}" + ) + else: return HttpResponse(f"Unknown action: {escape(action)}", status=400) diff --git a/netbox_librenms_plugin/views/imports/list.py b/netbox_librenms_plugin/views/imports/list.py index e0468f5abd..ca8ba15871 100644 --- a/netbox_librenms_plugin/views/imports/list.py +++ b/netbox_librenms_plugin/views/imports/list.py @@ -92,6 +92,8 @@ def _load_job_results(self, job_id): filters = job_data.get("filters", {}) server_key = job_data.get("server_key", "default") vc_enabled = job_data.get("vc_detection_enabled", False) + use_sysname = job_data.get("use_sysname", True) + strip_domain = job_data.get("strip_domain", False) # Extract cache metadata for frontend warnings self._cache_timestamp = job_data.get("cached_at") @@ -109,6 +111,8 @@ def _load_job_results(self, job_id): filters=filters, device_id=device_id, vc_enabled=vc_enabled, + use_sysname=use_sysname, + strip_domain=strip_domain, ) device = cache.get(cache_key) if device: @@ -259,9 +263,9 @@ def get(self, request, *args, **kwargs): # noqa: D401 - inherited doc return_cache_status=True, ) devices_cached = devices_from_cache - except Exception: + except Exception as e: # Cache check failed; proceed with background job decision based on device_count - pass + logger.debug("Cache check failed; proceeding without cached result: %s", e, exc_info=True) # Get device count for background job decision try: @@ -440,6 +444,8 @@ def _get_import_queryset(self): server_key=self.librenms_api.server_key, filters=libre_filters, vc_enabled=vc_detection_enabled, + use_sysname=self._use_sysname, + strip_domain=self._strip_domain, ) cache_metadata = cache.get(cache_metadata_key) if cache_metadata: diff --git a/netbox_librenms_plugin/views/object_sync/devices.py b/netbox_librenms_plugin/views/object_sync/devices.py index 97584f8319..429a3119a8 100644 --- a/netbox_librenms_plugin/views/object_sync/devices.py +++ b/netbox_librenms_plugin/views/object_sync/devices.py @@ -79,13 +79,22 @@ def get_redirect_url(self, obj): def get_table(self, data, obj, interface_name_field, vlan_groups=None): """Return the appropriate interface table, selecting VC variant if needed.""" + server_key = self.librenms_api.server_key if hasattr(obj, "virtual_chassis") and obj.virtual_chassis: table = VCInterfaceTable( - data, device=obj, interface_name_field=interface_name_field, vlan_groups=vlan_groups + data, + device=obj, + interface_name_field=interface_name_field, + vlan_groups=vlan_groups, + server_key=server_key, ) else: table = LibreNMSInterfaceTable( - data, device=obj, interface_name_field=interface_name_field, vlan_groups=vlan_groups + data, + device=obj, + interface_name_field=interface_name_field, + vlan_groups=vlan_groups, + server_key=server_key, ) table.htmx_url = f"{self.request.path}?tab=interfaces" return table diff --git a/netbox_librenms_plugin/views/object_sync/vms.py b/netbox_librenms_plugin/views/object_sync/vms.py index 51143d909d..bd6e052488 100644 --- a/netbox_librenms_plugin/views/object_sync/vms.py +++ b/netbox_librenms_plugin/views/object_sync/vms.py @@ -45,7 +45,9 @@ class VMInterfaceTableView(BaseInterfaceTableView): def get_table(self, data, obj, interface_name_field, vlan_groups=None): """Return a VM interface table for the given data.""" - return LibreNMSVMInterfaceTable(data, device=obj, vlan_groups=vlan_groups) + return LibreNMSVMInterfaceTable( + data, device=obj, vlan_groups=vlan_groups, server_key=self.librenms_api.server_key + ) def get_interfaces(self, obj): """Return all interfaces for the virtual machine.""" diff --git a/netbox_librenms_plugin/views/sync/device_fields.py b/netbox_librenms_plugin/views/sync/device_fields.py index 29eed67c2a..d621c7cec6 100644 --- a/netbox_librenms_plugin/views/sync/device_fields.py +++ b/netbox_librenms_plugin/views/sync/device_fields.py @@ -1,13 +1,16 @@ from dcim.models import Device, Manufacturer, Platform from django.contrib import messages from django.core.exceptions import ValidationError -from django.db import IntegrityError +from django.db import IntegrityError, transaction from django.shortcuts import get_object_or_404, redirect from django.views import View +import logging from netbox_librenms_plugin.utils import match_librenms_hardware_to_device_type from netbox_librenms_plugin.views.mixins import LibreNMSAPIMixin, LibreNMSPermissionMixin, NetBoxObjectPermissionMixin +logger = logging.getLogger(__name__) + class UpdateDeviceNameView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, LibreNMSAPIMixin, View): """Update NetBox device name from LibreNMS sysName.""" @@ -278,28 +281,43 @@ def post(self, request, pk): pass try: - platform = Platform.objects.create( - name=platform_name, - manufacturer=manufacturer, + with transaction.atomic(): + platform = Platform.objects.create( + name=platform_name, + manufacturer=manufacturer, + ) + + device.platform = platform + device.full_clean() + device.save() + except IntegrityError as e: + error_str = str(e) + logger.error( + f"IntegrityError creating platform '{platform_name}' for device pk={pk}: {e}", + exc_info=True, + ) + if "platform" in error_str.lower() or "slug" in error_str.lower(): + messages.error( + request, + f"Platform '{platform_name}' could not be created (slug collision). Try a different name.", + ) + else: + messages.error( + request, + f"Failed to assign platform '{platform_name}'. Please contact an administrator.", + ) + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + except ValidationError as e: + logger.error( + f"ValidationError assigning platform '{platform_name}' to device pk={pk}: {e}", + exc_info=True, ) - except IntegrityError: messages.error( request, - f"Platform '{platform_name}' could not be created (slug collision). Try a different name.", + f"Failed to assign platform '{platform_name}'. Please contact an administrator.", ) return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) - old_platform = device.platform - device.platform = platform - try: - device.full_clean() - device.save() - except (ValidationError, IntegrityError) as e: - device.platform = old_platform - error_msg = e.message_dict if hasattr(e, "message_dict") else str(e) - messages.error(request, f"Failed to assign platform '{platform}': {error_msg}") - return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) - messages.success( request, f"Created platform '{platform}' and assigned to device", @@ -382,3 +400,73 @@ def post(self, request, pk): messages.info(request, "No serial assignments were made") return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + +class RemoveServerMappingView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, View): + """Remove a single server entry from the device's librenms_id custom field dict.""" + + required_object_permissions = { + "POST": [("change", Device)], + } + + def post(self, request, pk): + if error := self.require_all_permissions("POST"): + return error + + device = get_object_or_404(Device, pk=pk) + server_key = request.POST.get("server_key", "").strip() + + if not server_key: + messages.error(request, "No server_key provided.") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + cf_value = device.custom_field_data.get("librenms_id") + if not isinstance(cf_value, dict) or server_key not in cf_value: + messages.warning(request, f"No mapping found for server '{server_key}'.") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + # Refuse to remove mappings for servers that are still configured in the plugin. + # Only orphaned (unconfigured) mappings may be removed via this endpoint. + # Guard both multi-server mode (servers dict) and legacy single-server mode + # (top-level librenms_url in plugin config, which implicitly defines "default"). + from django.conf import settings as django_settings + + plugins_cfg = django_settings.PLUGINS_CONFIG.get("netbox_librenms_plugin", {}) + configured_servers = plugins_cfg.get("servers", {}) + legacy_url_configured = bool(plugins_cfg.get("librenms_url")) + if server_key in configured_servers or (legacy_url_configured and server_key == "default"): + messages.error( + request, + f"Cannot remove mapping for configured server '{server_key}'. " + "Remove the server from plugin configuration first, then retry.", + ) + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + + with transaction.atomic(): + try: + device_locked = Device.objects.select_for_update().get(pk=pk) + except Device.DoesNotExist: + messages.error(request, "Device no longer exists.") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + cf = device_locked.custom_field_data.get("librenms_id", {}) + # Re-check after acquiring lock; mirror the pre-transaction protection logic + _is_protected = server_key in configured_servers or (legacy_url_configured and server_key == "default") + if isinstance(cf, dict) and server_key in cf and not _is_protected: + del cf[server_key] + device_locked.custom_field_data["librenms_id"] = cf if cf else None + try: + device_locked.full_clean() + device_locked.save() + except ValidationError as exc: + transaction.set_rollback(True) + messages.error(request, f"Validation error removing mapping: {exc}") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + except Exception as exc: + transaction.set_rollback(True) + messages.error(request, f"Error removing mapping for server '{server_key}': {exc}") + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) + messages.success(request, f"Removed LibreNMS mapping for server '{server_key}'.") + else: + messages.warning(request, f"Mapping for server '{server_key}' was already removed.") + + return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) diff --git a/netbox_librenms_plugin/views/sync/interfaces.py b/netbox_librenms_plugin/views/sync/interfaces.py index 1100e0da1b..f5d92ab0a2 100644 --- a/netbox_librenms_plugin/views/sync/interfaces.py +++ b/netbox_librenms_plugin/views/sync/interfaces.py @@ -9,7 +9,7 @@ from virtualization.models import VirtualMachine, VMInterface from netbox_librenms_plugin.models import InterfaceTypeMapping -from netbox_librenms_plugin.utils import convert_speed_to_kbps, get_interface_name_field +from netbox_librenms_plugin.utils import convert_speed_to_kbps, get_interface_name_field, set_librenms_device_id from netbox_librenms_plugin.views.mixins import ( CacheMixin, LibreNMSPermissionMixin, @@ -235,7 +235,9 @@ def update_interface_attributes( setattr(interface, netbox_key, librenms_interface.get(librenms_key)) if "librenms_id" in interface.cf: - interface.custom_field_data["librenms_id"] = librenms_interface.get("port_id") + port_id = librenms_interface.get("port_id") + if port_id is not None: + set_librenms_device_id(interface, port_id, self.librenms_api.server_key) if "enabled" not in exclude_columns: admin_status = librenms_interface.get("ifAdminStatus") @@ -245,7 +247,7 @@ def update_interface_attributes( else (admin_status.lower() == "up" if isinstance(admin_status, str) else bool(admin_status)) ) - if "mac_address" not in exclude_columns: + if "mac_address" not in exclude_columns and is_device_interface: ifPhysAddress = librenms_interface.get("ifPhysAddress") self.handle_mac_address(interface, ifPhysAddress) diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000000..32ce70698f --- /dev/null +++ b/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.12.0" + +[[package]] +name = "netbox-librenms-plugin" +version = "0.4.3" +source = { editable = "." } From cd041181d7910c8ecb2e04584c35ee89d7dfe5ae Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Wed, 4 Mar 2026 15:09:09 +0100 Subject: [PATCH 05/28] feat: add inventory/modules sync (rebased onto librenms_id + develop) Add ENTITY-MIB inventory sync for device modules (module types, module bays, transceivers, normalization rules) on top of the librenms_id JSON migration. Features added: - Module bay sync from LibreNMS ENTITY-MIB inventory data - ModuleBayMapping model for LibreNMS to NetBox bay name translation - ModuleTypeMapping for hardware model normalization - DeviceTypeMapping for device hardware matching - NormalizationRule model for generic regex-based string normalization - Module sync views (BaseModuleTableView, object sync tabs) - Transceiver detection via LibreNMS port transceiver API - Virtual chassis module support with per-member inventory - New migrations (0009-0013) for new models - New contrib/ YAML examples for mappings and rules - E2E test scaffolding in tests/e2e/ --- .devcontainer/README.md | 2 + .devcontainer/scripts/diagnose.sh | 1 + .devcontainer/scripts/load-aliases.sh | 1 + .devcontainer/scripts/setup.sh | 1 + .devcontainer/scripts/start-netbox.sh | 2 +- .devcontainer/scripts/welcome.sh | 3 +- .github/workflows/lint-format.yaml | 28 +- contrib/README.md | 28 + contrib/device_type_mappings.yaml | 73 ++ contrib/interface_name_rules.yaml | 200 ++++ contrib/interface_type_mappings.yaml | 70 ++ contrib/module_bay_mappings.yaml | 216 ++++ contrib/module_type_mappings.yaml | 332 +++++ contrib/normalization_rules.yaml | 61 + docs/usage_tips/custom_field.md | 8 +- docs/usage_tips/permissions.md | 2 +- netbox_librenms_plugin/__init__.py | 75 ++ netbox_librenms_plugin/api/serializers.py | 56 +- netbox_librenms_plugin/api/urls.py | 4 + netbox_librenms_plugin/api/views.py | 58 +- netbox_librenms_plugin/filters.py | 42 +- netbox_librenms_plugin/forms.py | 214 +++- .../import_utils/bulk_import.py | 9 +- .../import_utils/device_operations.py | 49 + netbox_librenms_plugin/librenms_api.py | 45 + .../migrations/0009_add_devicetypemapping.py | 45 + .../migrations/0010_add_moduletypemapping.py | 49 + .../migrations/0011_modulebaymapping.py | 38 + .../0012_add_is_regex_to_modulebaymapping.py | 18 + .../migrations/0013_normalizationrule.py | 93 ++ netbox_librenms_plugin/models.py | 203 ++++ netbox_librenms_plugin/navigation.py | 68 ++ .../js/librenms_sync.js | 4 + netbox_librenms_plugin/tables/mappings.py | 137 ++- netbox_librenms_plugin/tables/modules.py | 193 +++ .../_module_sync_content.html | 31 + .../devicetypemapping.html | 28 + .../devicetypemapping_list.html | 12 + .../inc/_module_sync.html | 27 + .../librenms_sync_base.html | 19 +- .../modulebaymapping.html | 30 + .../modulebaymapping_list.html | 12 + .../moduletypemapping.html | 28 + .../moduletypemapping_list.html | 12 + .../normalizationrule.html | 34 + .../normalizationrule_list.html | 16 + netbox_librenms_plugin/tests/test_init.py | 174 +++ netbox_librenms_plugin/tests/test_utils.py | 31 +- netbox_librenms_plugin/urls.py | 220 +++- netbox_librenms_plugin/utils.py | 143 ++- netbox_librenms_plugin/views/__init__.py | 34 + .../views/base/cables_view.py | 11 +- .../views/base/librenms_sync_view.py | 58 + .../views/base/modules_view.py | 1064 +++++++++++++++++ netbox_librenms_plugin/views/imports/list.py | 82 +- netbox_librenms_plugin/views/mapping_views.py | 276 ++++- .../views/object_sync/__init__.py | 1 + .../views/object_sync/devices.py | 20 + netbox_librenms_plugin/views/sync/cables.py | 29 +- netbox_librenms_plugin/views/sync/devices.py | 2 +- .../views/sync/interfaces.py | 6 - tests/e2e/__init__.py | 0 tests/e2e/conftest.py | 6 + tests/e2e/test_module_install.py | 330 +++++ 64 files changed, 5088 insertions(+), 76 deletions(-) create mode 100644 contrib/README.md create mode 100644 contrib/device_type_mappings.yaml create mode 100644 contrib/interface_name_rules.yaml create mode 100644 contrib/interface_type_mappings.yaml create mode 100644 contrib/module_bay_mappings.yaml create mode 100644 contrib/module_type_mappings.yaml create mode 100644 contrib/normalization_rules.yaml create mode 100644 netbox_librenms_plugin/migrations/0009_add_devicetypemapping.py create mode 100644 netbox_librenms_plugin/migrations/0010_add_moduletypemapping.py create mode 100644 netbox_librenms_plugin/migrations/0011_modulebaymapping.py create mode 100644 netbox_librenms_plugin/migrations/0012_add_is_regex_to_modulebaymapping.py create mode 100644 netbox_librenms_plugin/migrations/0013_normalizationrule.py create mode 100644 netbox_librenms_plugin/tables/modules.py create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping_list.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/inc/_module_sync.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping_list.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping_list.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule.html create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule_list.html create mode 100644 netbox_librenms_plugin/tests/test_init.py create mode 100644 netbox_librenms_plugin/views/base/modules_view.py create mode 100644 tests/e2e/__init__.py create mode 100644 tests/e2e/conftest.py create mode 100644 tests/e2e/test_module_install.py diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 3560a93af3..7024351758 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -98,6 +98,8 @@ Below are the dev container defaults. The field name to change these defaults is - Plugin loader: enabled; reads `.devcontainer/config/plugin-config.py` if present - If `plugin-config.py` is missing: plugin is enabled with empty config (features won’t work until configured) + + ## 🔧 Configuration ### NetBox Version and Environment (use .devcontainer/.env) diff --git a/.devcontainer/scripts/diagnose.sh b/.devcontainer/scripts/diagnose.sh index 133e6ca97f..be7596d699 100755 --- a/.devcontainer/scripts/diagnose.sh +++ b/.devcontainer/scripts/diagnose.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script echo "🔍 DevContainer Startup Diagnostics" echo "==================================" diff --git a/.devcontainer/scripts/load-aliases.sh b/.devcontainer/scripts/load-aliases.sh index 65149d6198..feac6ee98f 100755 --- a/.devcontainer/scripts/load-aliases.sh +++ b/.devcontainer/scripts/load-aliases.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script # Quick alias loader for current session # Usage: source .devcontainer/scripts/load-aliases.sh diff --git a/.devcontainer/scripts/setup.sh b/.devcontainer/scripts/setup.sh index 588fe652ab..7f4278fd46 100755 --- a/.devcontainer/scripts/setup.sh +++ b/.devcontainer/scripts/setup.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script set -e echo "🚀 Setting up NetBox LibreNMS Plugin development environment..." diff --git a/.devcontainer/scripts/start-netbox.sh b/.devcontainer/scripts/start-netbox.sh index 789dcb845a..d5e4796600 100755 --- a/.devcontainer/scripts/start-netbox.sh +++ b/.devcontainer/scripts/start-netbox.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script # Check if we should run in background or foreground BACKGROUND=false @@ -18,7 +19,6 @@ if [ "$CODESPACES" = "true" ] && [ -n "$CODESPACE_NAME" ]; then echo "🔗 GitHub Codespaces detected" else ACCESS_URL="http://localhost:8000" - echo "🐛 Debug: ACCESS_URL is set to: $ACCESS_URL" fi # Load shared process management helpers diff --git a/.devcontainer/scripts/welcome.sh b/.devcontainer/scripts/welcome.sh index 9328d663aa..e273313766 100755 --- a/.devcontainer/scripts/welcome.sh +++ b/.devcontainer/scripts/welcome.sh @@ -1,4 +1,5 @@ #!/bin/bash +# netbox-librenms-plugin devcontainer script # Ensure aliases are available in the postAttach terminal session source "$(dirname "$0")/load-aliases.sh" 2>/dev/null @@ -44,7 +45,7 @@ if [ -n "$CODESPACES" ]; then echo " 💡 Click the link in the Ports panel or look for the 'Open in Browser' button" else echo "🖥️ Local Development Environment:" - echo " NetBox will be available at: http://localhost:8000 (paste into you browser)" + echo " NetBox will be available at: http://localhost:8000 (paste into your browser)" fi echo "" diff --git a/.github/workflows/lint-format.yaml b/.github/workflows/lint-format.yaml index 3e12242f63..055f809cc5 100644 --- a/.github/workflows/lint-format.yaml +++ b/.github/workflows/lint-format.yaml @@ -2,13 +2,7 @@ name: Lint and Format on: push: - branches: - - master - - develop pull_request: - branches: - - master - - develop jobs: format-and-lint: @@ -20,8 +14,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: '3.9' - cache: 'pip' + python-version: '3.12' - name: Install dependencies run: | @@ -29,22 +22,7 @@ jobs: pip install ruff - name: Run Ruff linting - run: | - echo "::group::Ruff Linting" - ruff check . --output-format=github - echo "::endgroup::" + run: ruff check . - name: Run Ruff formatting check - run: | - echo "::group::Ruff Formatting" - ruff format --check . - echo "::endgroup::" - - - name: Report formatting issues - if: failure() - run: | - echo "::error::Formatting or linting issues detected!" - echo "To fix locally, run:" - echo " ruff check --fix ." - echo " ruff format ." - echo "Then commit and push the changes." + run: ruff format --check . diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000000..8714ac5342 --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,28 @@ +# Contrib: Example Mapping Files + +This directory contains example YAML mapping files for bulk import into the +NetBox LibreNMS Plugin. Each file can be imported via the plugin's bulk import +feature in the NetBox UI. + +## How to Import + +1. Navigate to the mapping page (e.g., **LibreNMS → Device Type Mappings**) +2. Click the **Import** button (upload icon) in the top right +3. Select **YAML** format +4. Paste the contents of the relevant YAML file +5. Click **Submit** + +## Available Mappings + +| File | Description | +|------|-------------| +| `interface_type_mappings.yaml` | Maps LibreNMS interface types + speeds to NetBox interface types | +| `device_type_mappings.yaml` | Maps LibreNMS hardware strings to NetBox device types | +| `module_type_mappings.yaml` | Maps LibreNMS inventory model names to NetBox module types (incl. transceivers) | +| `module_bay_mappings.yaml` | Maps LibreNMS inventory container names to NetBox module bay names | + +## Customisation + +These files are **examples** — adjust values to match the device types, module +types, and interface types defined in your NetBox instance. The `netbox_*` +fields must reference objects that already exist in your NetBox. diff --git a/contrib/device_type_mappings.yaml b/contrib/device_type_mappings.yaml new file mode 100644 index 0000000000..2dec241524 --- /dev/null +++ b/contrib/device_type_mappings.yaml @@ -0,0 +1,73 @@ +# Device Type Mappings +# +# Maps LibreNMS hardware strings to NetBox device types. +# Import via: LibreNMS Plugin > Device Type Mappings > Import +# +# Fields: +# librenms_hardware — Hardware string exactly as shown in LibreNMS +# netbox_device_type — NetBox DeviceType (matched by model name or ID) +# description — Optional note +# +# The librenms_hardware value is matched case-insensitively. +# These mappings are checked BEFORE the built-in part_number/model fallback. + +# Juniper — LibreNMS reports verbose marketing names +- librenms_hardware: "Juniper MX480 Internet Backbone Router" + netbox_device_type: "MX480" + description: "Juniper MX480 chassis" + +- librenms_hardware: "Juniper MX960 Internet Backbone Router" + netbox_device_type: "MX960" + description: "Juniper MX960 chassis" + +- librenms_hardware: "Juniper MX304 Edge Router" + netbox_device_type: "MX304" + description: "Juniper MX304 edge router" + +- librenms_hardware: "JNP10008 [PTX10008]" + netbox_device_type: "PTX10008" + description: "Juniper PTX10008 core router" + +- librenms_hardware: "JNP7100-32C [ACX7100-32C]" + netbox_device_type: "ACX7100-32C" + description: "Juniper ACX7100-32C" + +- librenms_hardware: "JNP7024 [ACX7024]" + netbox_device_type: "ACX7024" + description: "Juniper ACX7024" + +- librenms_hardware: "Juniper JNP10008 Internet Backbone Router" + netbox_device_type: "PTX10008" + description: "Juniper PTX10008 (alternate hardware string)" + +- librenms_hardware: "Juniper VRR Internet Backbone Router" + netbox_device_type: "VRR" + description: "Juniper Virtual Route Reflector" + +# Nokia — model string matches directly in most cases +- librenms_hardware: "7750 SR-7s" + netbox_device_type: "7750 SR-7s" + description: "Nokia 7750 SR-7s service router" + +# Cisco — often matches by part_number but not always +- librenms_hardware: "WS-C4900M" + netbox_device_type: "WS-C4900M" + description: "Cisco Catalyst 4900M" + +# Cisco IOS XR +- librenms_hardware: "8201-SYS" + netbox_device_type: "8201" + description: "Cisco 8201 (hardware string differs from model)" + +# UfiSpace — LibreNMS reports SONiC/ONIE platform names +- librenms_hardware: "x86-64-ufispace-s9610-36d-r0" + netbox_device_type: "S9610-36D" + description: "UfiSpace S9610-36D" + +- librenms_hardware: "x86-64-ufispace-s9610-46dx-r0" + netbox_device_type: "S9610-46DX" + description: "UfiSpace S9610-46DX" + +- librenms_hardware: "x86-64-ufispace-s9700-53dx-r9" + netbox_device_type: "S9700-53DX" + description: "UfiSpace S9700-53DX" diff --git a/contrib/interface_name_rules.yaml b/contrib/interface_name_rules.yaml new file mode 100644 index 0000000000..52da69dff5 --- /dev/null +++ b/contrib/interface_name_rules.yaml @@ -0,0 +1,200 @@ +# Interface Name Rules +# +# Post-install interface rename rules for module types where NetBox's +# position-based naming can't produce the correct interface name. +# +# Covers two scenarios: +# 1. Converter offset — e.g., GLC-T inside CVR-X2-SFP needs port numbering +# that accounts for the converter's position in the parent module bay. +# 2. Breakout channels — e.g., QSFP+ 4x10G produces multiple sub-interfaces +# from a single physical port. +# +# Template variables: +# {slot} — Top-level slot/module bay position +# {bay_position} — Position of the bay this module is installed into (raw) +# {bay_position_num} — Numeric suffix of bay position (e.g., "swp1" → "1") +# {parent_bay_position} — Position of the parent module's bay +# {sfp_slot} — Numeric sub-bay index within the parent module +# {base} — Original interface name from the NetBox module template +# {channel} — Breakout channel number (iterated) +# +# Arithmetic expressions are supported inside braces: +# {8 + ({parent_bay_position} - 1) * 2 + {sfp_slot}} +# +# Bulk import via: LibreNMS Plugin > Settings > Interface Name Rules > Import + +# --- Converter Offset Examples --- + +# SFP-1G-T (1G copper SFP, covers GLC-T/GLC-TE) in CVR-X2-SFP converter +# X2 bays are numbered 1-N; each converter holds 2 SFP slots +# Resulting interface: GigabitEthernet/ +- module_type: SFP-1G-T + parent_module_type: CVR-X2-SFP + name_template: "GigabitEthernet{slot}/{8 + ({parent_bay_position} - 1) * 2 + {sfp_slot}}" + channel_count: 0 + channel_start: 0 + description: "SFP-1G-T in CVR-X2-SFP: offset port numbering for X2-to-SFP conversion" + +# --- Breakout Channel Examples --- + +# QSFP-4X10G-LR breakout — Juniper-style (channels start at 0) +- module_type: QSFP-4X10G-LR + name_template: "{base}:{channel}" + channel_count: 4 + channel_start: 0 + description: "QSFP+ 4x10G-LR breakout with Juniper-style channel numbering (0-3)" + +# QSFP-4X10G-SR breakout — Juniper-style (channels start at 0) +- module_type: QSFP-4X10G-SR + name_template: "{base}:{channel}" + channel_count: 4 + channel_start: 0 + description: "QSFP+ 4x10G-SR breakout with Juniper-style channel numbering (0-3)" + +# --- Commented Examples --- + +# QSFP+ 4x10G breakout — Cisco-style (channels start at 1) +# - module_type: QSFP-4X10G-LR +# name_template: "{base}:{channel}" +# channel_count: 4 +# channel_start: 1 +# description: "QSFP+ 4x10G breakout with Cisco-style channel numbering (1-4)" + +# --- UfiSpace/Arcos Breakout Rules --- +# UfiSpace switches use swpNsC naming for breakout interfaces. +# bay_position_num extracts the numeric suffix from the bay name (e.g., "swp1" → "1"). +# Channels start at 1, with 2 channels per 100G QSFP28 (2x100G breakout). + +# S9610-36D breakout rules +- module_type: QSFP-100G-LR4 + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SR4 + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SWDM4 + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-ZR + device_type: S9610-36D + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" + +# S9610-46DX breakout rules +- module_type: QSFP-100G-LR4 + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SR4 + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SWDM4 + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-ZR + device_type: S9610-46DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" + +# S9700-53DX breakout rules +- module_type: QSFP-100G-LR4 + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SR4 + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-SWDM4 + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" +- module_type: QSFP-100G-ZR + device_type: S9700-53DX + name_template: "swp{bay_position_num}s{channel}" + channel_count: 2 + channel_start: 1 + description: "UfiSpace QSFP28 2x100G breakout" + +# --- Juniper ACX7024 Platform-Specific Rules --- +# These rules are scoped to the ACX7024 device type and use bay_position +# to generate Juniper-style interface names with FPC/PIC/port notation. + +# 100GE QSFP28 transceivers -> et-0/0/{port} +- module_type: QSFP-100G-LR4 + device_type: ACX7024 + name_template: "et-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 100GE QSFP28 naming" + +- module_type: QSFP-100G-SR4 + device_type: ACX7024 + name_template: "et-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 100GE QSFP28 naming" + +- module_type: QSFP-100G-SWDM4 + device_type: ACX7024 + name_template: "et-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 100GE QSFP28 naming" + +# 10GE SFP+ transceivers -> xe-0/0/{port} +- module_type: SFP-10G-SR + device_type: ACX7024 + name_template: "xe-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 10GE SFP+ naming" + +- module_type: SFP-10G-LR + device_type: ACX7024 + name_template: "xe-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 10GE SFP+ naming" + +# 1GE SFP transceivers -> ge-0/0/{port} +- module_type: SFP-1G-T + device_type: ACX7024 + name_template: "ge-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 1GE SFP naming" + +- module_type: SFP-1G-LX + device_type: ACX7024 + name_template: "ge-0/0/{bay_position}" + channel_count: 0 + channel_start: 0 + description: "Juniper ACX7024 1GE SFP naming" diff --git a/contrib/interface_type_mappings.yaml b/contrib/interface_type_mappings.yaml new file mode 100644 index 0000000000..19db2a1fcf --- /dev/null +++ b/contrib/interface_type_mappings.yaml @@ -0,0 +1,70 @@ +# Interface Type Mappings +# +# Maps LibreNMS interface types (and optional speeds) to NetBox interface types. +# Import via: LibreNMS Plugin > Interface Mappings > Import +# +# Fields: +# librenms_type — IANA ifType string from LibreNMS (e.g. ethernetCsmacd) +# librenms_speed — Speed in Kbps (optional, null matches any speed) +# netbox_type — NetBox InterfaceTypeChoices slug +# description — Optional note +# +# Common NetBox interface type slugs: +# 1000base-t, 10gbase-t, 10gbase-x-sfpp, 25gbase-x-sfp28, +# 40gbase-x-qsfpp, 100gbase-x-qsfp28, 400gbase-x-qsfpdd, +# ieee802.11ax, lag, virtual, other + +- librenms_type: ethernetCsmacd + librenms_speed: 1000000 + netbox_type: 1000base-t + description: "1G Ethernet copper" + +- librenms_type: ethernetCsmacd + librenms_speed: 10000000 + netbox_type: 10gbase-x-sfpp + description: "10G Ethernet SFP+" + +- librenms_type: ethernetCsmacd + librenms_speed: 25000000 + netbox_type: 25gbase-x-sfp28 + description: "25G Ethernet SFP28" + +- librenms_type: ethernetCsmacd + librenms_speed: 40000000 + netbox_type: 40gbase-x-qsfpp + description: "40G Ethernet QSFP+" + +- librenms_type: ethernetCsmacd + librenms_speed: 100000000 + netbox_type: 100gbase-x-qsfp28 + description: "100G Ethernet QSFP28" + +- librenms_type: ethernetCsmacd + librenms_speed: 400000000 + netbox_type: 400gbase-x-qsfpdd + description: "400G Ethernet QSFP-DD" + +- librenms_type: ieee8023adLag + librenms_speed: + netbox_type: lag + description: "LACP/LAG aggregation" + +- librenms_type: propVirtual + librenms_speed: + netbox_type: virtual + description: "Virtual/loopback interface" + +- librenms_type: softwareLoopback + librenms_speed: + netbox_type: virtual + description: "Software loopback" + +- librenms_type: tunnel + librenms_speed: + netbox_type: virtual + description: "Tunnel interface" + +- librenms_type: l2vlan + librenms_speed: + netbox_type: virtual + description: "VLAN interface" diff --git a/contrib/module_bay_mappings.yaml b/contrib/module_bay_mappings.yaml new file mode 100644 index 0000000000..64c063176a --- /dev/null +++ b/contrib/module_bay_mappings.yaml @@ -0,0 +1,216 @@ +# Module Bay Mappings - Map LibreNMS inventory container names to NetBox module bay names +# +# These mappings replace heuristic matching between LibreNMS inventory and NetBox module bays. +# Import via: LibreNMS Plugin → Module Bay Mappings → Import +# +# Fields: +# librenms_name: LibreNMS entPhysicalName or container name (exact match or regex) +# librenms_class: Optional entPhysicalClass filter (powerSupply, fan, module, etc.) +# Leave empty for class-independent mappings +# netbox_bay_name: Target NetBox module bay name (supports \1, \2 backreferences with regex) +# is_regex: Set to true to treat librenms_name as a Python regex pattern +# description: Optional description +# +# Regex patterns use Python re.fullmatch() — the pattern must match the entire string. +# Backreferences (\1, \2) in netbox_bay_name reference capture groups in the pattern. + +# ─── Regex Patterns ────────────────────────────────────────────────────────── +# These patterns replace many individual exact-match entries. + +# Arcos/UfiSpace: sfpN → Transceiver N (covers sfp0 through sfp53+) +- librenms_name: "^sfp(\\d+)$" + netbox_bay_name: "Transceiver \\1" + is_regex: true + description: "Arcos sfpN → Transceiver N" + +# Cisco X2: Port Container slot/port → X2 Port port +- librenms_name: "^Port Container (\\d+)/(\\d+)$" + netbox_bay_name: "X2 Port \\2" + is_regex: true + description: "Cisco X2 Port Container → X2 Port N" + +# Cisco modules: Linecard/Supervisor(slot N) → Slot N +- librenms_name: "^Linecard\\(slot (\\d+)\\)$" + librenms_class: "module" + netbox_bay_name: "Slot \\1" + is_regex: true + description: "Cisco Linecard slot → Slot N" +- librenms_name: "^Supervisor\\(slot (\\d+)\\)$" + librenms_class: "module" + netbox_bay_name: "Slot \\1" + is_regex: true + description: "Cisco Supervisor slot → Slot N" + +# Generic power supplies and fans +- librenms_name: "^Power Supply (\\d+)$" + librenms_class: "powerSupply" + netbox_bay_name: "PS\\1" + is_regex: true + description: "Power Supply N → PSN" +- librenms_name: "^FanTray (\\d+)$" + librenms_class: "fan" + netbox_bay_name: "Fan Tray \\1" + is_regex: true + description: "FanTray N → Fan Tray N" + +# Nokia 7750 SR chassis fans and power modules +- librenms_name: "^Chassis 1 Fan (\\d+)$" + librenms_class: "fan" + netbox_bay_name: "Fan \\1" + is_regex: true + description: "Nokia chassis fan → Fan N" +- librenms_name: "^Chassis 1 PowShelf 1 PM (\\d+)$" + librenms_class: "powerSupply" + netbox_bay_name: "PM \\1" + is_regex: true + description: "Nokia power module → PM N" + +# Nokia MDA and XIOM sub-module bays +# Bay names resolve from {module}/N templates: IOM Slot 1 pos=1 → bay {module}/1 = 1/1 +- librenms_name: "^MDA (\\d+)/(\\d+)$" + librenms_class: "mdaModule" + netbox_bay_name: "\\1/\\2" + is_regex: true + description: "Nokia MDA N/M → N/M (matches {module}/M on IOM)" +- librenms_name: "^XIOM (\\d+)/x(\\d+)$" + librenms_class: "xioModule" + netbox_bay_name: "\\1/x\\2" + is_regex: true + description: "Nokia XIOM N/xM → N/xM (matches {module}/xM on IOM)" +- librenms_name: "^MDA (\\d+)/x(\\d+)/(\\d+)$" + librenms_class: "mdaModule" + netbox_bay_name: "x\\2/\\3" + is_regex: true + description: "Nokia MDA in XIOM N/xP/Q → xP/Q (matches {module}/Q on XIOM)" + +# Nokia transceiver connector bays +# LibreNMS ifName "1/1/c1" (slot/mda/connector) → NetBox bay "1/c1" +# ({module} on MDA resolves to position, stripping the slot prefix) +- librenms_name: "(\\d+)/(\\d+)/(c\\d+)" + librenms_class: "port" + netbox_bay_name: "\\2/\\3" + is_regex: true + description: "Nokia transceiver slot/mda/cN → mda-pos/cN" +# LibreNMS ifName "2/x1/1/c2" (slot/xiom/mda/connector) → NetBox bay "1/c2" +- librenms_name: "(\\d+)/x(\\d+)/(\\d+)/(c\\d+)" + librenms_class: "port" + netbox_bay_name: "\\3/\\4" + is_regex: true + description: "Nokia XIOM transceiver slot/xiom/mda/cN → mda-pos/cN" + +# Juniper MX transceiver bays +# LibreNMS entPhysicalDescr format: "SFP+-10G-SR @ {fpc}/{pic}/{port}" +# NetBox MPC-3D-16XGE-SFPP bay format: "Transceiver {pic}/{port}" +- librenms_name: "[^@]+ @ \\d+/(\\d+)/(\\d+)" + librenms_class: "port" + netbox_bay_name: "Transceiver \\1/\\2" + is_regex: true + description: "Juniper MX SFP+ @ fpc/pic/port → Transceiver pic/port" + +# ─── Exact Match Entries ───────────────────────────────────────────────────── +# These are for special cases where names don't follow a regex pattern. + +# Nokia CPM slots +- librenms_name: "Slot A" + librenms_class: "cpmModule" + netbox_bay_name: "Slot A" + description: "Nokia CPM slot A" +- librenms_name: "Slot B" + librenms_class: "cpmModule" + netbox_bay_name: "Slot B" + description: "Nokia CPM slot B" +- librenms_name: "SR-7s 2 CPM mini" + librenms_class: "cpmCarrier" + netbox_bay_name: "CMA" + description: "Nokia CMA2-7s CPM carrier bracket" + +# Juniper fixed-form devices +- librenms_name: "PSM 0" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 0" + description: "Juniper PSU slot 0" +- librenms_name: "PSM 1" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 1" + description: "Juniper PSU slot 1" + +# Juniper chassis devices (PTX10008 etc.): PSM → PEM +# Regex runs after exact matches, so PSM 0/1 → PSU 0/1 above takes priority for ACX +- librenms_name: "^PSM (\\d+)$" + librenms_class: "powerSupply" + netbox_bay_name: "PEM \\1" + is_regex: true + description: "Juniper chassis PSM N → PEM N" + +# Juniper FPC container: "FPC: @ N/*/*" → FPC N +- librenms_name: "^FPC: .+ @ (\\d+)/\\*/\\*$" + librenms_class: "container" + netbox_bay_name: "FPC \\1" + is_regex: true + description: "Juniper FPC container description → FPC N" + +# Juniper transceivers: " @ slot/pic/port" description → Transceiver slot/pic/port +- librenms_name: "^.+ @ (\\d+/\\d+/\\d+)$" + librenms_class: "port" + netbox_bay_name: "Transceiver \\1" + is_regex: true + description: "Juniper transceiver description → Transceiver slot/pic/port" + +# Juniper fan trays: "Fan Tray N" → "Fan N" (ACX7100, etc.) +# Runs after exact match, so "Fan Tray 0" → "Fan Tray" (ACX7024) still works +- librenms_name: "^Fan Tray (\\d+)$" + librenms_class: "fan" + netbox_bay_name: "Fan \\1" + is_regex: true + description: "Juniper Fan Tray N → Fan N (ACX7100 etc.)" + +# Juniper MX304: PEM → PSU (MX304 bays are named PSU, not PEM) +- librenms_name: "PEM 0" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 0" + description: "Juniper MX304 PEM 0 → PSU 0" +- librenms_name: "PEM 1" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 1" + description: "Juniper MX304 PEM 1 → PSU 1" + +- librenms_name: "Fan Tray 0" + librenms_class: "fan" + netbox_bay_name: "Fan Tray" + description: "Juniper single fan tray (ACX7024)" + +# Juniper PTX10008: SIB → CB (Switch Interface Board → Component Board slot) +- librenms_name: "SIB 0" + librenms_class: "container" + netbox_bay_name: "CB 0" + description: "Juniper PTX10008 SIB 0 → CB 0" +- librenms_name: "SIB 1" + librenms_class: "container" + netbox_bay_name: "CB 1" + description: "Juniper PTX10008 SIB 1 → CB 1" +- librenms_name: "SIB 2" + librenms_class: "container" + netbox_bay_name: "CB 2" + description: "Juniper PTX10008 SIB 2 → CB 2" +- librenms_name: "SIB 3" + librenms_class: "container" + netbox_bay_name: "CB 3" + description: "Juniper PTX10008 SIB 3 → CB 3" +- librenms_name: "SIB 4" + librenms_class: "container" + netbox_bay_name: "CB 4" + description: "Juniper PTX10008 SIB 4 → CB 4" +- librenms_name: "SIB 5" + librenms_class: "container" + netbox_bay_name: "CB 5" + description: "Juniper PTX10008 SIB 5 → CB 5" + +# Arcos power supplies +- librenms_name: "psu0" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 0" + description: "Arcos PSU slot 0" +- librenms_name: "psu1" + librenms_class: "powerSupply" + netbox_bay_name: "PSU 1" + description: "Arcos PSU slot 1" diff --git a/contrib/module_type_mappings.yaml b/contrib/module_type_mappings.yaml new file mode 100644 index 0000000000..e70d726f1b --- /dev/null +++ b/contrib/module_type_mappings.yaml @@ -0,0 +1,332 @@ +# Module Type Mappings +# +# Maps LibreNMS inventory model names (entPhysicalModelName) to NetBox module types. +# Import via: LibreNMS Plugin > Module Type Mappings > Import +# +# Fields: +# librenms_model — Model name from LibreNMS SNMP inventory +# netbox_module_type — NetBox ModuleType (matched by model name or ID) +# description — Optional note +# +# These mappings are checked FIRST. If no mapping exists, the plugin falls back +# to exact model name and part_number matching against NetBox module types. + +# ─── Cisco Catalyst 4900M ──────────────────────────────────────────────────── + +- librenms_model: "WS-X4908-10GE" + netbox_module_type: "WS-X4908-10GE" + description: "Cisco 8-port 10G X2 line card" + +- librenms_model: "WS-X4992" + netbox_module_type: "WS-X4992" + description: "Cisco 48-port 10/100/1000 line card" + +- librenms_model: "PWR-C49M-1000AC" + netbox_module_type: "PWR-C49M-1000AC" + description: "Cisco 1000W AC power supply" + +- librenms_model: "CVR-X2-SFP" + netbox_module_type: "CVR-X2-SFP" + description: "Cisco X2-to-SFP converter" + +# ─── Juniper Backplane ─────────────────────────────────────────────────────── + +- librenms_model: "710-017414" + netbox_module_type: "MX480-CHASSIS-BP" + description: "Juniper MX480 backplane (matched by part number)" + +- librenms_model: "CHAS-BP-MX480-S" + netbox_module_type: "MX480-CHASSIS-BP" + description: "Juniper MX480 backplane (matched by name)" + +# ─── Juniper FPC / Line Card Mappings ──────────────────────────────────────── +# Juniper FPCs use 750-xxxxxx part numbers as entPhysicalModelName. + +- librenms_model: "750-018124" + netbox_module_type: "DPCE-R-4XGE-XFP" + description: "Juniper DPCE 4-port 10G XFP DPC" + +- librenms_model: "750-022765" + netbox_module_type: "DPCE-R-20GE-2XGE" + description: "Juniper DPCE 20x1G + 2x10G combo DPC" + +- librenms_model: "750-028467" + netbox_module_type: "MPC-3D-16XGE-SFPP" + description: "Juniper MPC 16-port 10G SFP+" + +- librenms_model: "750-056519" + netbox_module_type: "MPC7E-MRATE" + description: "Juniper MPC7E 12-port QSFP+/QSFP28 multirate" + +- librenms_model: "750-062581" + netbox_module_type: "MPC-3D-16XGE-SFPP" + description: "Juniper MPC 16-port 10G SFP+ (variant PN)" + +# ─── Juniper Power Supply Mappings ─────────────────────────────────────────── + +- librenms_model: "740-029970" + netbox_module_type: "PWR-MX480-2520-AC" + description: "Juniper MX480 2520W AC PSU" + +- librenms_model: "740-063046" + netbox_module_type: "PWR-MX480-2520-AC" + description: "Juniper MX480 2520W AC PSU (variant PN)" + +- librenms_model: "740-027760" + netbox_module_type: "PWR-MX960-4100-AC" + description: "Juniper MX960 4100W AC PSU" + +- librenms_model: "740-110419" + netbox_module_type: "JNP-PWR2200-AC" + description: "Juniper MX304 2200W AC PSU" + +# Removed: JPSU-1600W-1UACAFO — exact model match, no mapping needed + +# ─── Juniper Fan Tray Mappings ─────────────────────────────────────────────── + +- librenms_model: "740-031521" + netbox_module_type: "FFANTRAY-MX960-HC" + description: "Juniper MX960 high-capacity fan tray" + +- librenms_model: "760-126744" + netbox_module_type: "JNP-FAN-2RU" + description: "Juniper MX304 2RU fan tray" + +# Removed: JNP7100-FAN1RU-AO — exact model match, no mapping needed + +# ─── Nokia 7750 SR-7s Module Mappings ──────────────────────────────────────── +# Nokia 3HE part numbers are handled by NormalizationRule: +# 1. Strip extra text (e.g. "3HE10550AARA01 NOK IPU3BFUEAA" → "3HE10550AARA01") +# 2. Strip revision suffix (e.g. "3HE10550AARA01" → "3HE10550AA") +# The normalized value matches the part_number field on NetBox ModuleTypes. +# No explicit Nokia mappings are needed. + +# ─── Transceiver Mappings: Juniper Part Numbers ───────────────────────────── +# Juniper-qualified optics use 740-xxxxxx part numbers regardless of OEM vendor. + +- librenms_model: "740-013111" + netbox_module_type: "SFP-1G-T" + description: "Juniper SFP 1000BASE-T copper" + +- librenms_model: "740-021308" + netbox_module_type: "SFP-10G-SR" + description: "Juniper SFP+ 10G-SR" + +- librenms_model: "740-031850" + netbox_module_type: "SFP-1G-LX" + description: "Juniper SFP 1000BASE-LX 10km" + +- librenms_model: "740-031980" + netbox_module_type: "SFP-10G-SR" + description: "Juniper SFP+ 10G-SR" + +- librenms_model: "740-031981" + netbox_module_type: "SFP-10G-LR" + description: "Juniper SFP+ 10G-LR" + +- librenms_model: "740-047682" + netbox_module_type: "CFP-100G-LR4" + description: "Juniper CFP 100G-LR4" + +- librenms_model: "740-054050" + netbox_module_type: "QSFP-4X10G-LR" + description: "Juniper QSFP+ 4x10G-LR" + +- librenms_model: "740-054053" + netbox_module_type: "QSFP-4X10G-SR" + description: "Juniper QSFP+ 4x10G-SR" + +- librenms_model: "740-058732" + netbox_module_type: "QSFP-100G-LR4" + description: "Juniper QSFP28 100G-LR4" + +- librenms_model: "740-061405" + netbox_module_type: "QSFP-100G-SR4" + description: "Juniper QSFP28 100G-SR4" + +- librenms_model: "740-061409" + netbox_module_type: "QSFP-100G-LR4" + description: "Juniper QSFP28 100G-LR4" + +- librenms_model: "740-079871" + netbox_module_type: "QSFP28-DD-2X100G-LR4" + description: "Juniper QSFP-DD 2x100G-LR4" + +- librenms_model: "740-082823" + netbox_module_type: "QSFP-DD-400G-LR8" + description: "Juniper QSFP-DD 400G-LR8" + +- librenms_model: "740-085349" + netbox_module_type: "QSFP-DD-400G-FR4" + description: "Juniper QSFP-DD 400G-FR4" + +- librenms_model: "740-085351" + netbox_module_type: "QSFP-DD-400G-DR4" + description: "Juniper QSFP-DD 400G-DR4" + +- librenms_model: "740-096176" + netbox_module_type: "QSFP-DD-400G-LR4" + description: "Juniper QSFP-DD 400G-LR4 (10km variant)" + +- librenms_model: "740-131169" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Juniper QSFP-DD 400G-ZR-M" + +- librenms_model: "740-151745" + netbox_module_type: "QSFP-DD-400G-ZR-M-HP" + description: "Juniper QSFP-DD 400G-ZR-M high-power" + +- librenms_model: "740-172665" + netbox_module_type: "QSFP-100G-ZR" + description: "Juniper QSFP28 100G-ZR" + +# ─── Transceiver Mappings: Finisar / II-VI / Coherent ──────────────────────── +# These are BASE part numbers (after normalization strips customer suffixes). +# See contrib/normalization_rules.yaml for the Finisar suffix-stripping rule. + +- librenms_model: "FTLC1154RDPL" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4" + +- librenms_model: "FTLC1151RDPL" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4 (variant)" + +- librenms_model: "FTLX1474D3BCL" + netbox_module_type: "SFP-10G-LR" + description: "Finisar SFP+ 10G-LR" + +- librenms_model: "FTCD3323R1PCL" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Finisar/II-VI QSFP-DD 400G-ZR-M coherent" + +- librenms_model: "FTLC9152RGPL" + netbox_module_type: "QSFP-100G-SWDM4" + description: "Finisar QSFP28 100G-SWDM4" + +# ─── Transceiver Mappings: Cisco / Cisco-branded OEM ───────────────────────── + +- librenms_model: "X2-10GB-LR" + netbox_module_type: "X2-10GB-LR" + description: "Cisco X2 10G-LR" + +- librenms_model: "X2-10GB-SR" + netbox_module_type: "X2-10GB-SR" + description: "Cisco X2 10G-SR" + +- librenms_model: "GLC-T" + netbox_module_type: "SFP-1G-T" + description: "Cisco SFP 1000BASE-T copper" + +- librenms_model: "GLC-TE" + netbox_module_type: "SFP-1G-T" + description: "Cisco SFP 1000BASE-T copper (extended temp)" + +- librenms_model: "SPP5200LR-C5" + netbox_module_type: "SFP-10G-LR" + description: "Cisco-branded Sumitomo SFP+ 10G-LR" + +- librenms_model: "SPP5310LR-C5" + netbox_module_type: "SFP-10G-LR" + description: "Cisco-branded Sumitomo SFP+ 10G-LR" + +- librenms_model: "SFBR-709SMZ-CS1" + netbox_module_type: "SFP-10G-SR" + description: "Cisco-branded Avago/Broadcom SFP+ 10G-SR" + +- librenms_model: "DP04QSDD-HE0" + netbox_module_type: "QSFP-DD-400G-ZR+" + description: "Cisco/Acacia QSFP-DD 400G-ZR+ coherent" + +- librenms_model: "QDD-400G-ZRP-S" + netbox_module_type: "QSFP-DD-400G-ZR+" + description: "Cisco QSFP-DD 400G-ZR+" + +- librenms_model: "QDD-400G-ZR4-S" + netbox_module_type: "QSFP-DD-400G-ZR" + description: "Cisco QSFP-DD 400G-ZR" + +# ─── Transceiver Mappings: Ciena ───────────────────────────────────────────── + +- librenms_model: "180-3530-900" + netbox_module_type: "QSFP-DD-400G-ZR" + description: "Ciena WaveLogic 5 Nano QSFP-DD 400ZR" + +- librenms_model: "176-3360-900" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Ciena QSFP-DD 400G-ZR-M coherent" + +- librenms_model: "176-3530-901" + netbox_module_type: "QSFP-DD-400G-ZR" + description: "Ciena QSFP-DD 400G-ZR coherent" + +- librenms_model: "176-3590-900" + netbox_module_type: "QSFP-DD-400G-ZR-M" + description: "Ciena QSFP-DD 400G-ZR-M coherent" + +# ─── Transceiver Mappings: T1 Nexus ───────────────────────────────────────── + +- librenms_model: "T1-QDD-400G-LR4" + netbox_module_type: "QSFP-DD-400G-LR4" + description: "T1 Nexus QSFP-DD 400G-LR4" + +- librenms_model: "T1-QDD-400G-FR4" + netbox_module_type: "QSFP-DD-400G-FR4" + description: "T1 Nexus QSFP-DD 400G-FR4" + +- librenms_model: "T1-QSFP28-LR4" + netbox_module_type: "QSFP-100G-LR4" + description: "T1 Nexus QSFP28 100G-LR4" + +- librenms_model: "100G-LR4_A3" + netbox_module_type: "QSFP-100G-LR4" + description: "T1 Nexus QSFP28 100G-LR4 (rev A3)" + +# ─── Transceiver Mappings: Innolight ──────────────────────────────────────── + +- librenms_model: "T-DQ4CNT-NCN" + netbox_module_type: "QSFP-DD-400G-FR4" + description: "Innolight QSFP-DD 400G-FR4" + +# ─── Transceiver Mappings: FS.com ──────────────────────────────────────────── + +- librenms_model: "Q28-PC03" + netbox_module_type: "QSFP28-100G-CU3M" + description: "FS.com QSFP28 100G passive DAC 3m" + +# ─── Transceiver Mappings: ProLabs ─────────────────────────────────────────── + +- librenms_model: "Q28LR431-10-IN" + netbox_module_type: "QSFP-100G-LR4" + description: "ProLabs QSFP28 100G-LR4 10km" + +# ─── Transceiver Mappings: Arcos Fixed-Port Part Numbers ───────────────────── + +- librenms_model: "SP7041-TE" + netbox_module_type: "SFP-1G-T" + description: "SFP 1000BASE-T copper (Arcos platform)" + +# ─── Transceiver Mappings: LeGrand Innolight ───────────────────────────────── + +- librenms_model: "LGI-FTLC9152RGPL" + netbox_module_type: "QSFP-100G-SWDM4" + description: "LeGrand-branded Finisar QSFP28 100G-SWDM4" + +# ─── Transceiver Mappings: Additional Finisar Variants ────────────────────── +# Some transceivers have customer-code suffixes that normalization may not handle. +# Add direct mappings as fallback. + +- librenms_model: "FTLC1151RDPL-CN" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4 (CN customer code)" + +- librenms_model: "FTLC1154RDPL-A5" + netbox_module_type: "QSFP-100G-LR4" + description: "Finisar QSFP28 100G-LR4 (A5 customer code)" + +# ─── Unknown / Unidentified Part Numbers ───────────────────────────────────── +# These are mapped based on port context (QSFP28 100G slot) when vendor is unknown. + +- librenms_model: "1F3QAA" + netbox_module_type: "QSFP-100G-LR4" + description: "Unknown QSFP28 100G (mapped by port context)" diff --git a/contrib/normalization_rules.yaml b/contrib/normalization_rules.yaml new file mode 100644 index 0000000000..c3d2081bea --- /dev/null +++ b/contrib/normalization_rules.yaml @@ -0,0 +1,61 @@ +# Normalization Rules — Examples +# +# Regex-based string transformations applied before module type, device type, +# or module bay matching. Rules run in priority order (lower first); each +# rule's output feeds the next. +# +# Import via: LibreNMS → Normalization Rules → Import → YAML +# +# Fields: +# scope — module_type, device_type, or module_bay +# manufacturer — Optional manufacturer name (must exist in NetBox). +# When set, the rule only fires for that manufacturer. +# match_pattern — Python regex (re.sub pattern) +# replacement — Replacement string (supports \1, \2 back-references) +# priority — Lower values run first (default 100) +# description — Optional note + +# ── Nokia revision suffix stripping ────────────────────────────────────────── +# Nokia ENTITY-MIB reports module/transceiver models with 4-char revision +# suffixes (e.g. 3HE16474AARA01). NetBox module types use the base part +# number (3HE16474AA). This rule strips the suffix before matching. +# +# Captures the 10-char base (3HE + 5 alnum + 2 quality-tier letters), +# discards the 2-letter revision code + 2-digit build number. +- scope: module_type + manufacturer: Nokia + match_pattern: "^(3HE\\w{5}[A-Z]{2})[A-Z]{2}\\d{2}$" + replacement: "\\1" + priority: 100 + description: "Strip Nokia revision suffixes (e.g. RA01, RB01, RG01) from ENTITY-MIB model strings" + +# ── Finisar / II-VI / Coherent suffix stripping ───────────────────────────── +# Finisar part numbers have customer-specific suffixes after a hyphen: +# FTLC1154RDPL-A5 (original Finisar) +# FTLC1154RDPL-C (Prolabs compatible) +# FTLX1474D3BCL-C1 (Cisco-coded Finisar) +# This rule strips everything after the last hyphen for FT... models. +- scope: module_type + match_pattern: "^(FT[A-Z0-9]+)-[A-Z0-9]+$" + replacement: "\\1" + priority: 100 + description: "Strip Finisar/II-VI customer suffixes (-A5, -C, -CN, -C1, etc.)" + +# ── Prolabs LGI- prefix stripping ─────────────────────────────────────────── +# Prolabs-compatible optics sometimes prepend LGI- to the OEM part number: +# LGI-FTLC9152RGPL → FTLC9152RGPL +- scope: module_type + match_pattern: "^LGI-(.+)$" + replacement: "\\1" + priority: 50 + description: "Strip Prolabs LGI- prefix from OEM part numbers" + +# ── Nokia transceiver model field cleanup ──────────────────────────────────── +# Nokia transceiver API sometimes returns model strings with trailing vendor +# info: "3HE10550AARA01 NOK IPU3BFUEAA" — extract just the part number. +- scope: module_type + manufacturer: Nokia + match_pattern: "^(3HE\\w+)\\s+.*$" + replacement: "\\1" + priority: 50 + description: "Extract Nokia part number from transceiver model field (strip trailing vendor/oui info)" diff --git a/docs/usage_tips/custom_field.md b/docs/usage_tips/custom_field.md index 032812ed82..7ed27a2f97 100644 --- a/docs/usage_tips/custom_field.md +++ b/docs/usage_tips/custom_field.md @@ -4,6 +4,9 @@ To enhance device identification and synchronization between NetBox and LibreNMS, this plugin supports using a custom field `librenms_id` on Device, Virtual Machine and Interface objects. While the plugin works without it, using this custom field is recommended for LibreNMS API lookups, and to assist with matching the remote device and remote interfaces for cable creation in Netbox. It can also be entered manually if no primary IP or FQDN is available. +!!! info "Automatic Creation" + As of version 0.4.2, the plugin **automatically creates** the `librenms_id` custom field when migrations are run. You no longer need to create it manually. The field is created for Device, Virtual Machine, Interface, and VM Interface objects. + For the Device and Virtual Machine objects the plugin will automatically populate the LibreNMS ID custom field when opening the LibreNMS Sync page if the device has been found in LibreNMS. For the Interface object, the plugin will automatically populate the LibreNMS ID custom field when the interface data is synced from LibreNMS. @@ -15,7 +18,10 @@ For the Interface object, the plugin will automatically populate the LibreNMS ID - **Efficient Synchronization:** Enhances the reliability of API lookups. - **Cable creation:** Allows better device identification for the creation of cables between NetBox devices. -## Suggested Custom Field Setup +## Manual Custom Field Setup (Legacy) + +!!! note + This section is only needed if you are running an older version of the plugin that does not auto-create the field, or if you need to recreate it after deletion. Follow these steps to create the `librenms_id` custom field in NetBox: diff --git a/docs/usage_tips/permissions.md b/docs/usage_tips/permissions.md index 9f9ecfb3f3..39c5225d2a 100644 --- a/docs/usage_tips/permissions.md +++ b/docs/usage_tips/permissions.md @@ -26,7 +26,7 @@ A user needs both tiers of permissions to complete an action. For example, to vi The Plugin also enforces Netbox object permissions so the following permission would also be required: -2. **Tier 2: Object permission**: User needs `dcim.add_device` (to create the device in NetBox) +1. **Tier 2: Object permission**: User needs `dcim.add_device` (to create the device in NetBox) If either permission is missing, the operation fails with an appropriate error message. diff --git a/netbox_librenms_plugin/__init__.py b/netbox_librenms_plugin/__init__.py index 96b9997506..6b60735762 100644 --- a/netbox_librenms_plugin/__init__.py +++ b/netbox_librenms_plugin/__init__.py @@ -28,6 +28,7 @@ def ready(self): super().ready() from django.conf import settings + from django.db.models.signals import post_migrate plugin_config = getattr(settings, "PLUGINS_CONFIG", {}).get(self.name, {}) @@ -37,6 +38,12 @@ def ready(self): else: self._validate_legacy_config(plugin_config) + # Auto-create the librenms_id custom field after migrations complete + post_migrate.connect( + _ensure_librenms_id_custom_field, + dispatch_uid="netbox_librenms_plugin_ensure_cf", + ) + def _validate_multi_server_config(self, servers_config): """Validate multi-server configuration.""" if not servers_config or not isinstance(servers_config, dict): @@ -61,4 +68,72 @@ def _validate_legacy_config(self, plugin_config): ) +def _ensure_librenms_id_custom_field(sender, **kwargs): + """ + Auto-create (or migrate) the 'librenms_id' custom field. + Runs after migrations via post_migrate signal to ensure tables exist. + Uses dispatch_uid to avoid duplicate connections. + + librenms_id stores a per-server JSON mapping {"server_key": device_id}. + Legacy installations may have this field typed as 'integer'; we upgrade it + to 'json' automatically so the UI and API accept the dict format. + """ + # Only run once per migrate invocation (post_migrate fires per-app). + if getattr(_ensure_librenms_id_custom_field, "_executed", False): + return + + import logging + + try: + from django.contrib.contenttypes.models import ContentType + + from extras.models import CustomField + + cf, created = CustomField.objects.get_or_create( + name="librenms_id", + defaults={ + "type": "json", + "label": "LibreNMS ID", + "description": "LibreNMS Device ID for synchronization (auto-created by plugin)", + "required": False, + "ui_visible": "if-set", + "ui_editable": "yes", + "is_cloneable": False, + }, + ) + + # Migrate legacy integer-typed field to JSON so the multi-server + # dict format {"server_key": device_id} is accepted by the UI/API. + if not created and cf.type == "integer": + cf.type = "json" + cf.save(update_fields=["type"]) + logging.getLogger("netbox_librenms_plugin").info( + "Migrated 'librenms_id' custom field type from integer to json" + ) + + # Ensure the field is assigned to the required object types + from dcim.models import Device, Interface + from virtualization.models import VirtualMachine, VMInterface + + required_models = [Device, VirtualMachine, Interface, VMInterface] + current_types = set(cf.object_types.values_list("pk", flat=True)) + + for model in required_models: + ct = ContentType.objects.get_for_model(model) + if ct.pk not in current_types: + cf.object_types.add(ct) + + if created: + logging.getLogger("netbox_librenms_plugin").info( + "Auto-created 'librenms_id' custom field for Device, VirtualMachine, Interface, VMInterface" + ) + + # Only mark as executed after successful completion to allow retry on failure. + _ensure_librenms_id_custom_field._executed = True + except Exception as e: + # Don't break startup if custom field creation fails (e.g., during initial migration), + # but log the error so it's not silently swallowed. + logging.getLogger("netbox_librenms_plugin").exception("Failed to auto-create 'librenms_id' custom field: %s", e) + + config = LibreNMSSyncConfig diff --git a/netbox_librenms_plugin/api/serializers.py b/netbox_librenms_plugin/api/serializers.py index 6bcd0aef20..bcde788d2b 100644 --- a/netbox_librenms_plugin/api/serializers.py +++ b/netbox_librenms_plugin/api/serializers.py @@ -1,6 +1,12 @@ from netbox.api.serializers import NetBoxModelSerializer -from netbox_librenms_plugin.models import InterfaceTypeMapping +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) class InterfaceTypeMappingSerializer(NetBoxModelSerializer): @@ -11,3 +17,51 @@ class Meta: model = InterfaceTypeMapping fields = ["id", "librenms_type", "librenms_speed", "netbox_type", "description"] + + +class DeviceTypeMappingSerializer(NetBoxModelSerializer): + """Serialize DeviceTypeMapping model for REST API.""" + + class Meta: + """Meta options for DeviceTypeMappingSerializer.""" + + model = DeviceTypeMapping + fields = ["id", "librenms_hardware", "netbox_device_type", "description"] + + +class ModuleTypeMappingSerializer(NetBoxModelSerializer): + """Serialize ModuleTypeMapping model for REST API.""" + + class Meta: + """Meta options for ModuleTypeMappingSerializer.""" + + model = ModuleTypeMapping + fields = ["id", "librenms_model", "netbox_module_type", "description"] + + +class ModuleBayMappingSerializer(NetBoxModelSerializer): + """Serialize ModuleBayMapping model for REST API.""" + + class Meta: + """Meta options for ModuleBayMappingSerializer.""" + + model = ModuleBayMapping + fields = ["id", "librenms_name", "librenms_class", "netbox_bay_name", "is_regex", "description"] + + +class NormalizationRuleSerializer(NetBoxModelSerializer): + """Serialize NormalizationRule model for REST API.""" + + class Meta: + """Meta options for NormalizationRuleSerializer.""" + + model = NormalizationRule + fields = [ + "id", + "scope", + "manufacturer", + "match_pattern", + "replacement", + "priority", + "description", + ] diff --git a/netbox_librenms_plugin/api/urls.py b/netbox_librenms_plugin/api/urls.py index 230aa078d0..c032e7b2f5 100644 --- a/netbox_librenms_plugin/api/urls.py +++ b/netbox_librenms_plugin/api/urls.py @@ -7,6 +7,10 @@ router = NetBoxRouter() router.register("interface-type-mappings", views.InterfaceTypeMappingViewSet) +router.register("device-type-mappings", views.DeviceTypeMappingViewSet) +router.register("module-type-mappings", views.ModuleTypeMappingViewSet) +router.register("module-bay-mappings", views.ModuleBayMappingViewSet) +router.register("normalization-rules", views.NormalizationRuleViewSet) urlpatterns = [ path("jobs//sync-status/", views.sync_job_status, name="sync_job_status"), diff --git a/netbox_librenms_plugin/api/views.py b/netbox_librenms_plugin/api/views.py index 768c67f5fe..287a3858c1 100644 --- a/netbox_librenms_plugin/api/views.py +++ b/netbox_librenms_plugin/api/views.py @@ -11,9 +11,21 @@ from rq.job import Job as RQJob from netbox_librenms_plugin.constants import PERM_CHANGE_PLUGIN, PERM_VIEW_PLUGIN -from netbox_librenms_plugin.models import InterfaceTypeMapping - -from .serializers import InterfaceTypeMappingSerializer +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) + +from .serializers import ( + DeviceTypeMappingSerializer, + InterfaceTypeMappingSerializer, + ModuleBayMappingSerializer, + ModuleTypeMappingSerializer, + NormalizationRuleSerializer, +) logger = logging.getLogger(__name__) @@ -22,8 +34,8 @@ class LibreNMSPluginPermission(BasePermission): """ Permission class for LibreNMS plugin API endpoints. - - GET requests require view_librenmssettings - - All other requests require change_librenmssettings + - Safe requests (GET, HEAD, OPTIONS) require netbox_librenms_plugin.view_librenmssettings + - All other requests require netbox_librenms_plugin.change_librenmssettings """ def has_permission(self, request, view): @@ -41,6 +53,42 @@ class InterfaceTypeMappingViewSet(NetBoxModelViewSet): serializer_class = InterfaceTypeMappingSerializer +class DeviceTypeMappingViewSet(NetBoxModelViewSet): + """API viewset for DeviceTypeMapping CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = DeviceTypeMapping.objects.all() + serializer_class = DeviceTypeMappingSerializer + + +class ModuleTypeMappingViewSet(NetBoxModelViewSet): + """API viewset for ModuleTypeMapping CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = ModuleTypeMapping.objects.all() + serializer_class = ModuleTypeMappingSerializer + + +class ModuleBayMappingViewSet(NetBoxModelViewSet): + """API viewset for ModuleBayMapping CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = ModuleBayMapping.objects.all() + serializer_class = ModuleBayMappingSerializer + + +class NormalizationRuleViewSet(NetBoxModelViewSet): + """API viewset for NormalizationRule CRUD operations.""" + + permission_classes = [LibreNMSPluginPermission] + + queryset = NormalizationRule.objects.all() + serializer_class = NormalizationRuleSerializer + + @api_view(["POST"]) @permission_classes([LibreNMSPluginPermission]) def sync_job_status(request, job_pk): diff --git a/netbox_librenms_plugin/filters.py b/netbox_librenms_plugin/filters.py index 9ec162a64c..134bd8962d 100644 --- a/netbox_librenms_plugin/filters.py +++ b/netbox_librenms_plugin/filters.py @@ -1,6 +1,6 @@ import django_filters -from .models import InterfaceTypeMapping +from .models import DeviceTypeMapping, InterfaceTypeMapping, ModuleBayMapping, ModuleTypeMapping, NormalizationRule class InterfaceTypeMappingFilterSet(django_filters.FilterSet): @@ -11,3 +11,43 @@ class Meta: model = InterfaceTypeMapping fields = ["librenms_type", "librenms_speed", "netbox_type", "description"] + + +class DeviceTypeMappingFilterSet(django_filters.FilterSet): + """Filter set for DeviceTypeMapping model.""" + + class Meta: + """Meta options for DeviceTypeMappingFilterSet.""" + + model = DeviceTypeMapping + fields = ["librenms_hardware", "description"] + + +class ModuleTypeMappingFilterSet(django_filters.FilterSet): + """Filter set for ModuleTypeMapping model.""" + + class Meta: + """Meta options for ModuleTypeMappingFilterSet.""" + + model = ModuleTypeMapping + fields = ["librenms_model", "description"] + + +class ModuleBayMappingFilterSet(django_filters.FilterSet): + """Filter set for ModuleBayMapping model.""" + + class Meta: + """Meta options for ModuleBayMappingFilterSet.""" + + model = ModuleBayMapping + fields = ["librenms_name", "librenms_class", "netbox_bay_name", "is_regex"] + + +class NormalizationRuleFilterSet(django_filters.FilterSet): + """Filter set for NormalizationRule model.""" + + class Meta: + """Meta options for NormalizationRuleFilterSet.""" + + model = NormalizationRule + fields = ["scope", "manufacturer"] diff --git a/netbox_librenms_plugin/forms.py b/netbox_librenms_plugin/forms.py index d33e94d4ca..9c0bc7d9dc 100644 --- a/netbox_librenms_plugin/forms.py +++ b/netbox_librenms_plugin/forms.py @@ -2,7 +2,7 @@ import logging from dcim.choices import InterfaceTypeChoices -from dcim.models import Device, DeviceRole, DeviceType, Location, Rack, Site +from dcim.models import Device, DeviceRole, DeviceType, Location, Manufacturer, ModuleType, Rack, Site from django import forms from django.http import QueryDict from django.utils.translation import gettext_lazy as _ @@ -12,10 +12,22 @@ NetBoxModelImportForm, ) from netbox.plugins import get_plugin_config -from utilities.forms.fields import CSVChoiceField, DynamicModelMultipleChoiceField +from utilities.forms.fields import ( + CSVChoiceField, + CSVModelChoiceField, + DynamicModelChoiceField, + DynamicModelMultipleChoiceField, +) from virtualization.models import Cluster, VirtualMachine -from .models import InterfaceTypeMapping, LibreNMSSettings +from .models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + LibreNMSSettings, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) logger = logging.getLogger(__name__) @@ -51,11 +63,24 @@ def _get_librenms_poller_group_choices(): """ Helper function to get poller group choices from LibreNMS API. Shared between AddToLIbreSNMPV1V2 and AddToLIbreSNMPV3 forms. + Results are cached to avoid repeated API calls on every form instantiation. """ + from django.core.cache import cache + from .librenms_api import LibreNMSAPI choices = [("0", "Default (0)")] + try: + api = LibreNMSAPI() + server_id = api.librenms_url.rstrip("/") + cache_key = f"librenms_poller_group_choices_{server_id}" + except Exception: + cache_key = "librenms_poller_group_choices" + cached_choices = cache.get(cache_key) + if cached_choices: + return cached_choices + try: api = LibreNMSAPI() success, poller_groups = api.get_poller_groups() @@ -72,6 +97,8 @@ def _get_librenms_poller_group_choices(): else: label = f"{group_name} ({group_id})" choices.append((group_id, label)) + + cache.set(cache_key, choices, timeout=api.cache_timeout) except Exception: logger.exception("Failed to fetch LibreNMS poller groups; using default choices") @@ -90,10 +117,13 @@ class ServerConfigForm(NetBoxModelForm): ) class Meta: + """Meta options for ServerConfigForm.""" + model = LibreNMSSettings fields = ["selected_server"] def __init__(self, *args, **kwargs): + """Initialize form and populate server choices.""" super().__init__(*args, **kwargs) self.fields["selected_server"].choices = _get_librenms_server_choices() @@ -131,6 +161,8 @@ class ImportSettingsForm(NetBoxModelForm): ) class Meta: + """Meta options for ImportSettingsForm.""" + model = LibreNMSSettings fields = [ "vc_member_name_pattern", @@ -213,6 +245,8 @@ class InterfaceTypeMappingForm(NetBoxModelForm): """ class Meta: + """Meta options for InterfaceTypeMappingForm.""" + model = InterfaceTypeMapping fields = ["librenms_type", "librenms_speed", "netbox_type", "description"] @@ -230,6 +264,8 @@ class InterfaceTypeMappingImportForm(NetBoxModelImportForm): ) class Meta: + """Meta options for InterfaceTypeMappingImportForm.""" + model = InterfaceTypeMapping fields = ["librenms_type", "librenms_speed", "netbox_type", "description"] @@ -260,6 +296,175 @@ class InterfaceTypeMappingFilterForm(NetBoxModelFilterSetForm): model = InterfaceTypeMapping +class DeviceTypeMappingForm(NetBoxModelForm): + """Form for creating and editing device type mappings between LibreNMS and NetBox.""" + + netbox_device_type = forms.ModelChoiceField( + queryset=DeviceType.objects.all(), + label="NetBox Device Type", + widget=forms.Select(attrs={"class": "form-select"}), + ) + + class Meta: + """Meta options for DeviceTypeMappingForm.""" + + model = DeviceTypeMapping + fields = ["librenms_hardware", "netbox_device_type", "description"] + + +class DeviceTypeMappingImportForm(NetBoxModelImportForm): + """Form for bulk importing device type mappings.""" + + netbox_device_type = CSVModelChoiceField( + queryset=DeviceType.objects.all(), + to_field_name="model", + help_text="NetBox device type model name", + ) + + class Meta: + """Meta options for DeviceTypeMappingImportForm.""" + + model = DeviceTypeMapping + fields = ["librenms_hardware", "netbox_device_type", "description"] + + +class DeviceTypeMappingFilterForm(NetBoxModelFilterSetForm): + """Form for filtering device type mappings.""" + + librenms_hardware = forms.CharField(required=False, label="LibreNMS Hardware") + description = forms.CharField( + required=False, + label="Description", + help_text="Filter by description (partial match)", + ) + + model = DeviceTypeMapping + + +class ModuleTypeMappingForm(NetBoxModelForm): + """Form for creating and editing module type mappings between LibreNMS and NetBox.""" + + class Meta: + """Meta options for ModuleTypeMappingForm.""" + + model = ModuleTypeMapping + fields = ["librenms_model", "netbox_module_type", "description"] + + +class ModuleTypeMappingImportForm(NetBoxModelImportForm): + """Form for bulk importing module type mappings.""" + + netbox_module_type = CSVModelChoiceField( + queryset=ModuleType.objects.all(), + to_field_name="model", + help_text="NetBox module type model name", + ) + + class Meta: + """Meta options for ModuleTypeMappingImportForm.""" + + model = ModuleTypeMapping + fields = ["librenms_model", "netbox_module_type", "description"] + + +class ModuleTypeMappingFilterForm(NetBoxModelFilterSetForm): + """Form for filtering module type mappings.""" + + librenms_model = forms.CharField(required=False, label="LibreNMS Model") + description = forms.CharField( + required=False, + label="Description", + help_text="Filter by description (partial match)", + ) + + model = ModuleTypeMapping + + +class ModuleBayMappingForm(NetBoxModelForm): + """Form for creating and editing module bay mappings between LibreNMS and NetBox.""" + + class Meta: + """Meta options for ModuleBayMappingForm.""" + + model = ModuleBayMapping + fields = ["librenms_name", "librenms_class", "netbox_bay_name", "is_regex", "description"] + + +class ModuleBayMappingImportForm(NetBoxModelImportForm): + """Form for bulk importing module bay mappings.""" + + class Meta: + """Meta options for ModuleBayMappingImportForm.""" + + model = ModuleBayMapping + fields = ["librenms_name", "librenms_class", "netbox_bay_name", "is_regex", "description"] + + +class ModuleBayMappingFilterForm(NetBoxModelFilterSetForm): + """Form for filtering module bay mappings.""" + + librenms_name = forms.CharField(required=False, label="LibreNMS Name") + librenms_class = forms.CharField(required=False, label="LibreNMS Class") + netbox_bay_name = forms.CharField(required=False, label="NetBox Bay Name") + is_regex = forms.NullBooleanField(required=False, label="Regex") + + model = ModuleBayMapping + + +class NormalizationRuleForm(NetBoxModelForm): + """Form for creating and editing normalization rules.""" + + manufacturer = DynamicModelChoiceField( + queryset=Manufacturer.objects.all(), + required=False, + help_text="Optional: scope this rule to a specific manufacturer", + ) + + class Meta: + """Meta options for NormalizationRuleForm.""" + + model = NormalizationRule + fields = ["scope", "manufacturer", "match_pattern", "replacement", "priority", "description"] + + +class NormalizationRuleImportForm(NetBoxModelImportForm): + """Form for bulk importing normalization rules.""" + + scope = CSVChoiceField( + choices=NormalizationRule.SCOPE_CHOICES, + help_text="Scope: module_type, device_type, or module_bay", + ) + manufacturer = CSVModelChoiceField( + queryset=Manufacturer.objects.all(), + to_field_name="name", + required=False, + help_text="Optional manufacturer name (must already exist in NetBox)", + ) + + class Meta: + """Meta options for NormalizationRuleImportForm.""" + + model = NormalizationRule + fields = ["scope", "manufacturer", "match_pattern", "replacement", "priority", "description"] + + +class NormalizationRuleFilterForm(NetBoxModelFilterSetForm): + """Form for filtering normalization rules.""" + + scope = forms.ChoiceField( + required=False, + choices=[("", "---------")] + NormalizationRule.SCOPE_CHOICES, + label="Scope", + ) + manufacturer_id = DynamicModelChoiceField( + queryset=Manufacturer.objects.all(), + required=False, + label="Manufacturer", + ) + + model = NormalizationRule + + class AddToLIbreSNMPV1V2(forms.Form): """ Form for adding devices to LibreNMS using SNMPv1 or SNMPv2c authentication. @@ -315,6 +520,7 @@ class AddToLIbreSNMPV1V2(forms.Form): ) def __init__(self, *args, **kwargs): + """Initialize form and populate poller group choices.""" super().__init__(*args, **kwargs) self.fields["poller_group"].choices = _get_librenms_poller_group_choices() @@ -412,6 +618,7 @@ class AddToLIbreSNMPV3(forms.Form): ) def __init__(self, *args, **kwargs): + """Initialize form and populate poller group choices.""" super().__init__(*args, **kwargs) self.fields["poller_group"].choices = _get_librenms_poller_group_choices() @@ -422,6 +629,7 @@ class DeviceStatusFilterForm(NetBoxModelFilterSetForm): """ def __init__(self, *args, **kwargs): + """Initialize form and remove saved filter field.""" super().__init__(*args, **kwargs) # Remove the saved filter field if it exists if "filter_id" in self.fields: diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 845ceca0c1..21105cb3bb 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -377,8 +377,15 @@ def _refresh_existing_device(validation: dict, libre_device: dict = None, server validation["existing_match_type"] = match_type validation["can_import"] = False validation["is_ready"] = False + if not import_as_vm and hasattr(new_device, "role") and new_device.role: + validation["device_role"] = {"found": True, "role": new_device.role} except Exception as e: - logger.error(f"Failed to re-check for imported device: {e}") + logger.error(f"Failed to check for newly imported device: {e}") + + +def _empty_return(return_cache_status: bool): + """Centralised empty-result return value for process_device_filters.""" + return ([], False) if return_cache_status else [] def _empty_return(return_cache_status: bool): diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index 0420b87447..487c915cf1 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -26,6 +26,44 @@ logger = logging.getLogger(__name__) +def _try_chassis_device_type_match(api, device_id): + """ + Attempt device type matching using chassis inventory fields. + + When the LibreNMS hardware string doesn't match any NetBox device type, + the chassis entity often contains a more standardized identifier + (e.g., entPhysicalName 'CHAS-BP-MX480-S' or entPhysicalModelName '710-017414') + that matches a DeviceType part_number or model. + + Tries entPhysicalName first (typically the chassis part number), + then entPhysicalModelName as fallback. + + Returns: + dict with matched/device_type/match_type keys, or None on failure. + """ + skip_values = {"", "-", "Unspecified", "BUILTIN", "None"} + + try: + success, inventory = api.get_inventory_filtered(device_id, ent_physical_class="chassis") + if not success or not inventory: + return None + + for item in inventory: + # Try entPhysicalName first (often the chassis part number like CHAS-BP-MX480-S) + for field in ("entPhysicalName", "entPhysicalModelName"): + value = item.get(field) or "" + if value and value not in skip_values: + chassis_match = match_librenms_hardware_to_device_type(value) + if chassis_match["matched"]: + chassis_match["match_type"] = "chassis" + chassis_match["chassis_model"] = value + return chassis_match + except Exception: + logger.debug(f"Chassis inventory fallback failed for device {device_id}", exc_info=True) + + return None + + def _determine_device_name( libre_device: dict, use_sysname: bool = True, @@ -470,12 +508,23 @@ def validate_device_for_import( # 3. Validate DeviceType (required) hardware = libre_device.get("hardware", "") dt_match = match_librenms_hardware_to_device_type(hardware) + + # Chassis inventory fallback: when hardware doesn't match, + # try the chassis entPhysicalModelName as an additional lookup source + if not dt_match["matched"] and api: + device_id = libre_device.get("device_id") + if device_id: + chassis_match = _try_chassis_device_type_match(api, device_id) + if chassis_match and chassis_match["matched"]: + dt_match = chassis_match + # Update result keys individually to preserve the existing schema (especially "found") result["device_type"]["found"] = dt_match["matched"] result["device_type"]["device_type"] = dt_match.get("device_type") result["device_type"]["match_type"] = dt_match.get("match_type") if not dt_match["matched"]: + result["device_type"]["found"] = False result["issues"].append(f"No matching device type found for hardware: '{hardware}'") # Get some device types for user to choose from all_device_types = DeviceType.objects.all()[:10] diff --git a/netbox_librenms_plugin/librenms_api.py b/netbox_librenms_plugin/librenms_api.py index 8e2b2b3ac0..3291aad0cc 100644 --- a/netbox_librenms_plugin/librenms_api.py +++ b/netbox_librenms_plugin/librenms_api.py @@ -690,6 +690,51 @@ def get_device_inventory(self, device_id): except requests.exceptions.RequestException as e: return False, str(e) + def get_device_transceivers(self, device_id): + """ + Fetch all transceiver data for a device from LibreNMS. + + Route: /api/v0/devices/{device_id}/transceivers + + This is a separate data source from entity inventory. Some vendors + (e.g., Nokia/SROS) don't expose SFPs via ENTITY-MIB but do report + them through vendor-specific MIBs which LibreNMS surfaces here. + + Args: + device_id: LibreNMS device ID + + Returns: + tuple: (success: bool, data: list) + + Example transceiver item: + { + "port_id": 519, + "entity_physical_index": 1610899520, + "type": "CFP2/QSFP28", + "model": "3HE10550AARA01", + "serial": "X42AU0D", + "channels": 4, + "connector": "LC", + "wavelength": 1301, + ... + } + """ + try: + response = requests.get( + f"{self.librenms_url}/api/v0/devices/{device_id}/transceivers", + headers=self.headers, + timeout=DEFAULT_API_TIMEOUT, + verify=self.verify_ssl, + ) + response.raise_for_status() + + if response.status_code == 200: + data = response.json() + return True, data.get("transceivers", []) + return False, [] + except requests.exceptions.RequestException as e: + return False, str(e) + def get_poller_groups(self): """ Fetch all poller groups from LibreNMS. diff --git a/netbox_librenms_plugin/migrations/0009_add_devicetypemapping.py b/netbox_librenms_plugin/migrations/0009_add_devicetypemapping.py new file mode 100644 index 0000000000..dcd8fc4fd5 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0009_add_devicetypemapping.py @@ -0,0 +1,45 @@ +# Generated by Django 5.2.10 on 2026-02-17 11:48 + +import django.db.models.deletion +import netbox.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("dcim", "0225_gfk_indexes"), + ("extras", "0134_owner"), + ("netbox_librenms_plugin", "0008_librenmssettings_import_defaults"), + ] + + operations = [ + migrations.CreateModel( + name="DeviceTypeMapping", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ("librenms_hardware", models.CharField(max_length=255, unique=True)), + ("description", models.TextField(blank=True)), + ( + "netbox_device_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="librenms_mappings", + to="dcim.devicetype", + ), + ), + ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), + ], + options={ + "ordering": ["librenms_hardware"], + }, + bases=(netbox.models.deletion.DeleteMixin, models.Model), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0010_add_moduletypemapping.py b/netbox_librenms_plugin/migrations/0010_add_moduletypemapping.py new file mode 100644 index 0000000000..796bbceafd --- /dev/null +++ b/netbox_librenms_plugin/migrations/0010_add_moduletypemapping.py @@ -0,0 +1,49 @@ +# Generated by Django 5.2.10 on 2026-02-17 12:23 + +import django.db.models.deletion +import netbox.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("dcim", "0225_gfk_indexes"), + ("extras", "0134_owner"), + ("netbox_librenms_plugin", "0009_add_devicetypemapping"), + ] + + operations = [ + migrations.AlterModelOptions( + name="interfacetypemapping", + options={"ordering": ["librenms_type", "librenms_speed"]}, + ), + migrations.CreateModel( + name="ModuleTypeMapping", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ("librenms_model", models.CharField(max_length=255, unique=True)), + ("description", models.TextField(blank=True)), + ( + "netbox_module_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="librenms_mappings", + to="dcim.moduletype", + ), + ), + ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), + ], + options={ + "ordering": ["librenms_model"], + }, + bases=(netbox.models.deletion.DeleteMixin, models.Model), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0011_modulebaymapping.py b/netbox_librenms_plugin/migrations/0011_modulebaymapping.py new file mode 100644 index 0000000000..5b3c2c3be0 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0011_modulebaymapping.py @@ -0,0 +1,38 @@ +# Generated by Django 5.2.10 on 2026-02-17 12:29 + +import netbox.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("extras", "0134_owner"), + ("netbox_librenms_plugin", "0010_add_moduletypemapping"), + ] + + operations = [ + migrations.CreateModel( + name="ModuleBayMapping", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ("librenms_name", models.CharField(max_length=255)), + ("librenms_class", models.CharField(blank=True, max_length=50)), + ("netbox_bay_name", models.CharField(max_length=255)), + ("description", models.TextField(blank=True)), + ("tags", taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag")), + ], + options={ + "ordering": ["librenms_name"], + "unique_together": {("librenms_name", "librenms_class")}, + }, + bases=(netbox.models.deletion.DeleteMixin, models.Model), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0012_add_is_regex_to_modulebaymapping.py b/netbox_librenms_plugin/migrations/0012_add_is_regex_to_modulebaymapping.py new file mode 100644 index 0000000000..52ff053e20 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0012_add_is_regex_to_modulebaymapping.py @@ -0,0 +1,18 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("netbox_librenms_plugin", "0011_modulebaymapping"), + ] + + operations = [ + migrations.AddField( + model_name="modulebaymapping", + name="is_regex", + field=models.BooleanField( + default=False, + help_text="Treat LibreNMS Name as a regex pattern with backreferences in NetBox Bay Name", + ), + ), + ] diff --git a/netbox_librenms_plugin/migrations/0013_normalizationrule.py b/netbox_librenms_plugin/migrations/0013_normalizationrule.py new file mode 100644 index 0000000000..71d1f80509 --- /dev/null +++ b/netbox_librenms_plugin/migrations/0013_normalizationrule.py @@ -0,0 +1,93 @@ +"""Restore NormalizationRule model. + +The table was created by earlier migrations (0013 + 0014 in a previous branch) +and already exists in the database. This migration uses SeparateDatabaseAndState +so Django's ORM knows about the model without trying to CREATE the table again. +If the table doesn't exist (fresh install), the database_operations handle creation. +""" + +import django.db.models.deletion +import taggit.managers +import utilities.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("dcim", "0001_initial"), + ("extras", "0001_initial"), + ("netbox_librenms_plugin", "0012_add_is_regex_to_modulebaymapping"), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.CreateModel( + name="NormalizationRule", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("last_updated", models.DateTimeField(auto_now=True, null=True)), + ( + "custom_field_data", + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ( + "scope", + models.CharField( + choices=[ + ("module_type", "Module Type"), + ("device_type", "Device Type"), + ("module_bay", "Module Bay"), + ], + max_length=50, + ), + ), + ("match_pattern", models.CharField(max_length=500)), + ("replacement", models.CharField(max_length=500)), + ("priority", models.PositiveIntegerField(default=100)), + ("description", models.TextField(blank=True)), + ( + "manufacturer", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="normalization_rules", + to="dcim.manufacturer", + ), + ), + ( + "tags", + taggit.managers.TaggableManager(through="extras.TaggedItem", to="extras.Tag"), + ), + ], + options={ + "ordering": ["scope", "priority", "pk"], + }, + ), + ], + database_operations=[ + migrations.RunSQL( + sql=""" + CREATE TABLE IF NOT EXISTS "netbox_librenms_plugin_normalizationrule" ( + "id" bigserial NOT NULL PRIMARY KEY, + "created" timestamp with time zone NULL, + "last_updated" timestamp with time zone NULL, + "custom_field_data" jsonb NOT NULL DEFAULT '{}'::jsonb, + "scope" varchar(50) NOT NULL, + "match_pattern" varchar(500) NOT NULL, + "replacement" varchar(500) NOT NULL, + "priority" integer NOT NULL DEFAULT 100 CHECK ("priority" >= 0), + "description" text NOT NULL DEFAULT '', + "manufacturer_id" bigint NULL REFERENCES "dcim_manufacturer" ("id") + DEFERRABLE INITIALLY DEFERRED + ); + CREATE INDEX IF NOT EXISTS "netbox_librenms_plugin_norm_mfg_idx" + ON "netbox_librenms_plugin_normalizationrule" ("manufacturer_id"); + """, + reverse_sql="DROP TABLE IF EXISTS netbox_librenms_plugin_normalizationrule;", + ), + ], + ), + ] diff --git a/netbox_librenms_plugin/models.py b/netbox_librenms_plugin/models.py index cd79f47550..3978d9c5f4 100644 --- a/netbox_librenms_plugin/models.py +++ b/netbox_librenms_plugin/models.py @@ -1,4 +1,8 @@ +import re + from dcim.choices import InterfaceTypeChoices +from dcim.models import DeviceType, Manufacturer, ModuleType +from django.core.exceptions import ValidationError from django.db import models from django.urls import reverse from netbox.models import NetBoxModel @@ -71,6 +75,205 @@ class Meta: """Meta options for InterfaceTypeMapping.""" unique_together = ["librenms_type", "librenms_speed"] + ordering = ["librenms_type", "librenms_speed"] def __str__(self): return f"{self.librenms_type} + {self.librenms_speed} -> {self.netbox_type}" + + +class DeviceTypeMapping(NetBoxModel): + """Map LibreNMS hardware strings to NetBox DeviceType objects.""" + + librenms_hardware = models.CharField( + max_length=255, + unique=True, + help_text="Hardware string as reported by LibreNMS (e.g., 'Juniper MX480 Internet Backbone Router')", + ) + netbox_device_type = models.ForeignKey( + DeviceType, + on_delete=models.CASCADE, + related_name="librenms_mappings", + help_text="The NetBox DeviceType this hardware string maps to", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this mapping", + ) + + def get_absolute_url(self): + """Return the URL for this mapping's detail page.""" + return reverse("plugins:netbox_librenms_plugin:devicetypemapping_detail", args=[self.pk]) + + class Meta: + """Meta options for DeviceTypeMapping.""" + + ordering = ["librenms_hardware"] + + def __str__(self): + return f"{self.librenms_hardware} -> {self.netbox_device_type}" + + +class ModuleTypeMapping(NetBoxModel): + """Map LibreNMS inventory model names to NetBox ModuleType objects.""" + + librenms_model = models.CharField( + max_length=255, + unique=True, + help_text="Model name from LibreNMS inventory (entPhysicalModelName)", + ) + netbox_module_type = models.ForeignKey( + ModuleType, + on_delete=models.CASCADE, + related_name="librenms_mappings", + help_text="The NetBox ModuleType this model name maps to", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this mapping", + ) + + def get_absolute_url(self): + """Return the URL for this mapping's detail page.""" + return reverse("plugins:netbox_librenms_plugin:moduletypemapping_detail", args=[self.pk]) + + class Meta: + """Meta options for ModuleTypeMapping.""" + + ordering = ["librenms_model"] + + def __str__(self): + return f"{self.librenms_model} -> {self.netbox_module_type}" + + +class ModuleBayMapping(NetBoxModel): + """Map LibreNMS inventory names to NetBox module bay names. + + Used when LibreNMS inventory names don't match NetBox bay names exactly. + For example: LibreNMS "Power Supply 1" → NetBox "PS1". + When is_regex is True, librenms_name is treated as a regex pattern and + netbox_bay_name can use backreferences (\\1, \\2, etc.). + Mappings are global (not scoped to device type or manufacturer). + """ + + librenms_name = models.CharField( + max_length=255, + help_text="Name from LibreNMS inventory (entPhysicalName). " + "When 'Use Regex' is enabled, this is a Python regex pattern.", + ) + librenms_class = models.CharField( + max_length=50, + blank=True, + help_text="Optional entPhysicalClass filter (e.g. 'powerSupply', 'fan', 'module')", + ) + netbox_bay_name = models.CharField( + max_length=255, + help_text="NetBox module bay name to match. With regex, supports backreferences (\\1, \\2, etc.).", + ) + is_regex = models.BooleanField( + default=False, + help_text="Treat LibreNMS Name as a regex pattern with backreferences in NetBox Bay Name", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this mapping", + ) + + def clean(self): + """Validate that regex patterns compile when is_regex is True.""" + super().clean() + if self.is_regex: + try: + re.compile(self.librenms_name) + except re.error as e: + raise ValidationError({"librenms_name": f"Invalid regex: {e}"}) + + def get_absolute_url(self): + """Return the URL for this mapping's detail page.""" + return reverse("plugins:netbox_librenms_plugin:modulebaymapping_detail", args=[self.pk]) + + class Meta: + """Meta options for ModuleBayMapping.""" + + unique_together = ["librenms_name", "librenms_class"] + ordering = ["librenms_name"] + + def __str__(self): + cls = f" [{self.librenms_class}]" if self.librenms_class else "" + return f"{self.librenms_name}{cls} -> {self.netbox_bay_name}" + + +class NormalizationRule(NetBoxModel): + """Regex-based string normalization applied before matching lookups. + + Generic building block: a single rule engine handles normalization + for module types, device types, module bays, and future scopes. + Rules are applied in priority order; each transforms the string + for the next rule in the chain. + + Example – strip Nokia revision suffixes: + scope: module_type + match_pattern: ^(3HE\\w{5}[A-Z]{2})[A-Z]{2}\\d{2}$ + replacement: \\1 + Result: 3HE16474AARA01 → 3HE16474AA + """ + + SCOPE_MODULE_TYPE = "module_type" + SCOPE_DEVICE_TYPE = "device_type" + SCOPE_MODULE_BAY = "module_bay" + + SCOPE_CHOICES = [ + (SCOPE_MODULE_TYPE, "Module Type"), + (SCOPE_DEVICE_TYPE, "Device Type"), + (SCOPE_MODULE_BAY, "Module Bay"), + ] + + scope = models.CharField( + max_length=50, + choices=SCOPE_CHOICES, + help_text="Which matching lookup this rule applies to", + ) + manufacturer = models.ForeignKey( + Manufacturer, + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="normalization_rules", + help_text="Optional: only apply this rule to items from this manufacturer. " + "Leave blank for vendor-agnostic rules.", + ) + match_pattern = models.CharField( + max_length=500, + help_text="Regex pattern to match against input string (Python re syntax)", + ) + replacement = models.CharField( + max_length=500, + help_text="Replacement string (supports regex back-references \\1, \\2, …)", + ) + priority = models.PositiveIntegerField( + default=100, + help_text="Lower values run first. Rules chain: each transforms the output of the previous.", + ) + description = models.TextField( + blank=True, + help_text="Optional description or notes about this rule", + ) + + def clean(self): + """Validate that match_pattern compiles as a regex.""" + super().clean() + try: + re.compile(self.match_pattern) + except re.error as e: + raise ValidationError({"match_pattern": f"Invalid regex: {e}"}) + + def get_absolute_url(self): + """Return the URL for this rule's detail page.""" + return reverse("plugins:netbox_librenms_plugin:normalizationrule_detail", args=[self.pk]) + + class Meta: + """Meta options for NormalizationRule.""" + + ordering = ["scope", "priority", "pk"] + + def __str__(self): + return f"[{self.get_scope_display()}] {self.match_pattern} → {self.replacement}" diff --git a/netbox_librenms_plugin/navigation.py b/netbox_librenms_plugin/navigation.py index a08e62740f..052c06363c 100644 --- a/netbox_librenms_plugin/navigation.py +++ b/netbox_librenms_plugin/navigation.py @@ -31,6 +31,74 @@ ), ), ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:devicetypemapping_list", + link_text="Device Type Mappings", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:devicetypemapping_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:devicetypemapping_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:moduletypemapping_list", + link_text="Module Type Mappings", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:moduletypemapping_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:moduletypemapping_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:modulebaymapping_list", + link_text="Module Bay Mappings", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:modulebaymapping_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:modulebaymapping_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), + PluginMenuItem( + link="plugins:netbox_librenms_plugin:normalizationrule_list", + link_text="Normalization Rules", + permissions=[PERM_VIEW_PLUGIN], + buttons=( + PluginMenuButton( + link="plugins:netbox_librenms_plugin:normalizationrule_add", + title="Add", + icon_class="mdi mdi-plus-thick", + ), + PluginMenuButton( + link="plugins:netbox_librenms_plugin:normalizationrule_bulk_import", + title="Import", + icon_class="mdi mdi-upload", + ), + ), + ), ), ), ( diff --git a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js index cd470af1b1..0b42112b24 100644 --- a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js +++ b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js @@ -153,11 +153,15 @@ function initializeCountdowns() { if (window.vlanCountdownInterval) { clearInterval(window.vlanCountdownInterval); } + if (window.moduleCountdownInterval) { + clearInterval(window.moduleCountdownInterval); + } window.interfaceCountdownInterval = initializeCountdown("countdown-timer"); window.cableCountdownInterval = initializeCountdown("cable-countdown-timer"); window.ipCountdownInterval = initializeCountdown("ip-countdown-timer"); window.vlanCountdownInterval = initializeCountdown("vlan-countdown-timer"); + window.moduleCountdownInterval = initializeCountdown("module-countdown-timer"); } // ============================================ diff --git a/netbox_librenms_plugin/tables/mappings.py b/netbox_librenms_plugin/tables/mappings.py index 73949fd2c8..4b4b31a41d 100644 --- a/netbox_librenms_plugin/tables/mappings.py +++ b/netbox_librenms_plugin/tables/mappings.py @@ -1,7 +1,13 @@ import django_tables2 as tables from netbox.tables import NetBoxTable, columns -from netbox_librenms_plugin.models import InterfaceTypeMapping +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) class InterfaceTypeMappingTable(NetBoxTable): @@ -36,3 +42,132 @@ class Meta: "actions", ) attrs = {"class": "table table-hover table-headings table-striped"} + + +class DeviceTypeMappingTable(NetBoxTable): + """Table for displaying DeviceTypeMapping data.""" + + librenms_hardware = tables.Column(verbose_name="LibreNMS Hardware", linkify=True) + netbox_device_type = tables.Column(verbose_name="NetBox Device Type", linkify=True) + description = tables.Column(verbose_name="Description", linkify=False) + actions = columns.ActionsColumn(actions=("edit", "delete")) + + class Meta: + """Meta options for DeviceTypeMappingTable.""" + + model = DeviceTypeMapping + fields = ( + "id", + "librenms_hardware", + "netbox_device_type", + "description", + "actions", + ) + default_columns = ( + "id", + "librenms_hardware", + "netbox_device_type", + "description", + "actions", + ) + attrs = {"class": "table table-hover table-headings table-striped"} + + +class ModuleTypeMappingTable(NetBoxTable): + """Table for displaying ModuleTypeMapping data.""" + + librenms_model = tables.Column(verbose_name="LibreNMS Model", linkify=True) + netbox_module_type = tables.Column(verbose_name="NetBox Module Type", linkify=True) + description = tables.Column(verbose_name="Description", linkify=False) + actions = columns.ActionsColumn(actions=("edit", "delete")) + + class Meta: + """Meta options for ModuleTypeMappingTable.""" + + model = ModuleTypeMapping + fields = ( + "id", + "librenms_model", + "netbox_module_type", + "description", + "actions", + ) + default_columns = ( + "id", + "librenms_model", + "netbox_module_type", + "description", + "actions", + ) + attrs = {"class": "table table-hover table-headings table-striped"} + + +class ModuleBayMappingTable(NetBoxTable): + """Table for displaying ModuleBayMapping data.""" + + librenms_name = tables.Column(verbose_name="LibreNMS Name", linkify=True) + librenms_class = tables.Column(verbose_name="LibreNMS Class") + netbox_bay_name = tables.Column(verbose_name="NetBox Bay Name") + is_regex = columns.BooleanColumn(verbose_name="Regex") + description = tables.Column(verbose_name="Description", linkify=False) + actions = columns.ActionsColumn(actions=("edit", "delete")) + + class Meta: + """Meta options for ModuleBayMappingTable.""" + + model = ModuleBayMapping + fields = ( + "id", + "librenms_name", + "librenms_class", + "netbox_bay_name", + "is_regex", + "description", + "actions", + ) + default_columns = ( + "id", + "librenms_name", + "librenms_class", + "netbox_bay_name", + "is_regex", + "description", + "actions", + ) + attrs = {"class": "table table-hover table-headings table-striped"} + + +class NormalizationRuleTable(NetBoxTable): + """Table for displaying NormalizationRule data.""" + + scope = tables.Column(verbose_name="Scope", linkify=True) + manufacturer = tables.Column(verbose_name="Manufacturer", linkify=True) + match_pattern = tables.Column(verbose_name="Match Pattern") + replacement = tables.Column(verbose_name="Replacement") + priority = tables.Column(verbose_name="Priority") + description = tables.Column(verbose_name="Description", linkify=False) + actions = columns.ActionsColumn(actions=("edit", "delete")) + + class Meta: + """Meta options for NormalizationRuleTable.""" + + model = NormalizationRule + fields = ( + "id", + "scope", + "manufacturer", + "match_pattern", + "replacement", + "priority", + "description", + "actions", + ) + default_columns = ( + "id", + "scope", + "match_pattern", + "replacement", + "priority", + "actions", + ) + attrs = {"class": "table table-hover table-headings table-striped"} diff --git a/netbox_librenms_plugin/tables/modules.py b/netbox_librenms_plugin/tables/modules.py new file mode 100644 index 0000000000..5a4a6e7cb8 --- /dev/null +++ b/netbox_librenms_plugin/tables/modules.py @@ -0,0 +1,193 @@ +import django_tables2 as tables +from django.urls import reverse +from django.utils.html import format_html +from utilities.paginator import EnhancedPaginator + +from netbox_librenms_plugin.utils import get_table_paginate_count + + +class LibreNMSModuleTable(tables.Table): + """Table for displaying LibreNMS inventory items mapped to NetBox modules.""" + + name = tables.Column(verbose_name="Name", attrs={"td": {"data-col": "name"}}) + model = tables.Column(verbose_name="Model", attrs={"td": {"data-col": "model"}}) + serial = tables.Column(verbose_name="Serial", attrs={"td": {"data-col": "serial"}}) + description = tables.Column(verbose_name="Description", attrs={"td": {"data-col": "description"}}) + item_class = tables.Column(verbose_name="Class", attrs={"td": {"data-col": "item_class"}}) + module_bay = tables.Column(verbose_name="Module Bay", attrs={"td": {"data-col": "module_bay"}}) + module_type = tables.Column(verbose_name="Module Type", attrs={"td": {"data-col": "module_type"}}) + status = tables.Column(verbose_name="Status", attrs={"td": {"data-col": "status"}}) + actions = tables.Column( + verbose_name="Actions", orderable=False, empty_values=(), attrs={"td": {"data-col": "actions"}} + ) + + class Meta: + attrs = {"class": "table table-hover object-list", "id": "librenms-module-table"} + row_attrs = {"class": lambda record: record.get("row_class", "")} + + def __init__(self, *args, device=None, **kwargs): + """Initialize table with optional device context.""" + self.device = device + self.csrf_token = "" + super().__init__(*args, **kwargs) + self.tab = "modules" + self.htmx_url = None + self.prefix = "modules_" + + def configure(self, request): + """Configure pagination settings and CSRF token.""" + from django.middleware.csrf import get_token + + self.csrf_token = get_token(request) + paginate = {"paginator_class": EnhancedPaginator, "per_page": get_table_paginate_count(request, self.prefix)} + tables.RequestConfig(request, paginate).configure(self) + + def render_name(self, value, record): + """Render inventory item name with tree indentation for sub-components.""" + depth = record.get("depth", 0) + if depth == 0: + return value or "-" + # Build visual tree prefix based on nesting depth + padding_px = depth * 20 + prefix = "└─ " + return format_html('{}{}', padding_px, prefix, value or "-") + + def render_model(self, value, record): + """Render model with link to module type if matched.""" + if not value or value == "-": + return "-" + if url := record.get("module_type_url"): + return format_html('{}', url, value) + return value + + def render_serial(self, value, record): + """Render serial number.""" + return value or "-" + + def render_description(self, value, record): + """Render description, truncated for display.""" + if not value: + return "-" + if len(value) > 60: + return format_html('{}…', value, value[:57]) + return value + + def render_item_class(self, value, record): + """Render the entPhysicalClass with an icon.""" + icons = { + "module": "mdi-expansion-card", + "ioModule": "mdi-expansion-card", + "cpmModule": "mdi-expansion-card", + "mdaModule": "mdi-expansion-card", + "fabricModule": "mdi-expansion-card", + "xioModule": "mdi-expansion-card", + "powerSupply": "mdi-power-plug", + "fan": "mdi-fan", + "port": "mdi-ethernet", + "other": "mdi-card-outline", + } + icon = icons.get(value, "mdi-card-outline") + return format_html(' {}', icon, value) + + def render_module_bay(self, value, record): + """Render module bay with link if found in NetBox.""" + if not value or value == "-": + return format_html('No matching bay') + if url := record.get("module_bay_url"): + return format_html('{}', url, value) + return value + + def render_module_type(self, value, record): + """Render module type match status.""" + if not value or value == "-": + return format_html('No matching type') + if url := record.get("module_type_url"): + return format_html('{}', url, value) + return value + + def render_status(self, value, record): + """Render sync status with badge.""" + badge_classes = { + "Installed": "bg-success", + "Matched": "bg-info", + "No Bay": "bg-warning", + "No Type": "bg-warning", + "Unmatched": "bg-secondary", + "Serial Mismatch": "bg-danger", + "Name Conflict": "bg-warning", + } + badge_class = badge_classes.get(value, "bg-secondary") + if warning := record.get("module_path_warning"): + return format_html( + '{}' + ' ', + badge_class, + warning, + value, + "Upgrade NetBox to fully support {module_path}", + ) + if warning := record.get("name_conflict_warning"): + return format_html( + '{}' + ' ', + badge_class, + warning, + value, + warning, + ) + if hint := record.get("module_type_upgrade_hint"): + return format_html( + '{} ', + badge_class, + value, + hint, + ) + return format_html('{}', badge_class, value) + + def render_actions(self, value, record): + """Render install button for matched modules and install branch for parents.""" + if not self.device: + return "" + + buttons = [] + + # Single install button + if record.get("can_install"): + url = reverse("plugins:netbox_librenms_plugin:install_module", kwargs={"pk": self.device.pk}) + buttons.append( + format_html( + '
' + '' + '' + '' + '' + '
", + url, + self.csrf_token, + record.get("module_bay_id", ""), + record.get("module_type_id", ""), + record.get("serial", ""), + ) + ) + + # Install branch button for parents with installable children + if record.get("has_installable_children") and record.get("ent_physical_index"): + url = reverse("plugins:netbox_librenms_plugin:install_branch", kwargs={"pk": self.device.pk}) + buttons.append( + format_html( + '
' + '' + '' + '
", + url, + self.csrf_token, + record.get("ent_physical_index", ""), + ) + ) + + return format_html("{}", format_html("".join(str(b) for b in buttons))) if buttons else "" diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html new file mode 100644 index 0000000000..ce6e430bfa --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html @@ -0,0 +1,31 @@ +{% load helpers %} +{% include 'inc/messages.html' %} + + +{% if module_sync.table %} +
+
+ + Showing inventory items from LibreNMS matched against NetBox module bays and module types. + +
+ {% if module_sync.cache_expiry %} +
+ Cache expires in: +
+ {% endif %} +
+ +
+ {% include 'netbox_librenms_plugin/inc/paginator.html' with table=module_sync.table %} + {% include 'inc/table.html' with table=module_sync.table %} + {% include 'netbox_librenms_plugin/inc/paginator.html' with table=module_sync.table %} +
+{% else %} +
+
+ +

No inventory data loaded. Click Refresh Modules to fetch data from LibreNMS.

+
+
+{% endif %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping.html new file mode 100644 index 0000000000..3894441179 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping.html @@ -0,0 +1,28 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} + +{% block content %} +
+
+
+ + + + + + + + + + + + + + + +
LibreNMS HardwareNetBox Device TypeDescription
{{ object.librenms_hardware }}{{ object.netbox_device_type }}{{ object.description|default:"—" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping_list.html new file mode 100644 index 0000000000..06c95270b3 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/devicetypemapping_list.html @@ -0,0 +1,12 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Device Type Mapping

+

Map LibreNMS hardware strings to NetBox device types. + When importing devices from LibreNMS, these mappings are checked first before + falling back to exact part number / model matching.

+

Example: Map "Juniper MX480 Internet Backbone Router" to device type "MX480"

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/inc/_module_sync.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/inc/_module_sync.html new file mode 100644 index 0000000000..4c2c5ae65d --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/inc/_module_sync.html @@ -0,0 +1,27 @@ +{% load helpers %} + + +
+

Module Sync

+
+
+ {% csrf_token %} + {% if has_librenms_id %} + {% with model_name=object|meta:"model_name" %} + {% if model_name == "device" %} + + {% endif %} + {% endwith %} + {% endif %} +
+
+
+ + +
+ {% include 'netbox_librenms_plugin/_module_sync_content.html' %} +
diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html index 64ec26cb63..0350bf9dff 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html @@ -311,7 +311,7 @@
Device Information Sync
{{ object.name }}
- {% if sysName and sysName != object.name %} + {% if sysName and sysName != "-" and sysName != object.name %}
@@ -321,7 +321,7 @@
Device Information Sync
Sync to NetBox
- {% elif sysName %} + {% elif sysName and sysName != "-" %} @@ -597,6 +597,14 @@
Device Information Sync
{% endif %} {% endwith %} + {% if module_sync %} + + {% endif %}
Device Information Sync
{% include 'netbox_librenms_plugin/_ipaddress_sync.html' %} + {% if module_sync %} +
+ {% include 'netbox_librenms_plugin/inc/_module_sync.html' %} +
+ {% endif %} + {% with model_name=object|meta:"model_name" %} {% if model_name == "device" %}
+
+
+ + + + + + + + + + + + + + + + + +
LibreNMS NameLibreNMS ClassNetBox Bay NameDescription
{{ object.librenms_name }}{{ object.librenms_class|default:"—" }}{{ object.netbox_bay_name }}{{ object.description|default:"—" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping_list.html new file mode 100644 index 0000000000..fb87f901ec --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/modulebaymapping_list.html @@ -0,0 +1,12 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Module Bay Mapping

+

Map LibreNMS inventory container names to NetBox module bay names. + When synchronizing modules from LibreNMS, these mappings determine which + NetBox module bay a LibreNMS component should be installed into.

+

Example: Map "Linecard(slot 1)" to "Slot 1", or "Power Supply 1" to "PS1"

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping.html new file mode 100644 index 0000000000..019b0e51ed --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping.html @@ -0,0 +1,28 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} + +{% block content %} +
+
+
+ + + + + + + + + + + + + + + +
LibreNMS ModelNetBox Module TypeDescription
{{ object.librenms_model }}{{ object.netbox_module_type }}{{ object.description|default:"—" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping_list.html new file mode 100644 index 0000000000..4cfc22d592 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/moduletypemapping_list.html @@ -0,0 +1,12 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Module Type Mapping

+

Map LibreNMS inventory model names (entPhysicalModelName) to NetBox module types. + When synchronizing modules from LibreNMS, these mappings are checked first before + falling back to exact model / part number matching.

+

Example: Map "710-017414" to module type "WS-X4908-10GE"

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule.html new file mode 100644 index 0000000000..a1be7537a3 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule.html @@ -0,0 +1,34 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} + +{% block content %} +
+
+
+ + + + + + + + + + + + + + + + + + + + + +
ScopeManufacturerMatch PatternReplacementPriorityDescription
{{ object.get_scope_display }}{% if object.manufacturer %}{{ object.manufacturer }}{% else %}—{% endif %}{{ object.match_pattern }}{{ object.replacement }}{{ object.priority }}{{ object.description|default:"—" }}
+
+
+
+{% endblock %} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule_list.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule_list.html new file mode 100644 index 0000000000..d543141680 --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/normalizationrule_list.html @@ -0,0 +1,16 @@ +{% extends 'generic/object_list.html' %} + +{% block content %} +
+

Normalization Rules

+

Regex-based string normalization applied before matching lookups. + When a LibreNMS string doesn't match any NetBox object or mapping entry, + normalization rules transform it (e.g. strip revision suffixes) and retry.

+

Rules are chained in priority order per scope. One rule engine serves + module types, device types, and module bays.

+

Example — strip Nokia revision suffixes:
+ ^(3HE\w{5}[A-Z]{2})[A-Z]{2}\d{2}$ → \1
+ Turns 3HE16474AARA01 into 3HE16474AA which matches the part number.

+
+ {{ block.super }} +{% endblock %} diff --git a/netbox_librenms_plugin/tests/test_init.py b/netbox_librenms_plugin/tests/test_init.py new file mode 100644 index 0000000000..426f6736c0 --- /dev/null +++ b/netbox_librenms_plugin/tests/test_init.py @@ -0,0 +1,174 @@ +"""Tests for netbox_librenms_plugin.__init__ module. + +Covers the _ensure_librenms_id_custom_field post_migrate signal handler. +""" + +from unittest.mock import MagicMock, patch + + +# ============================================================================= +# TestEnsureLibreNMSIdCustomField - 6 tests +# ============================================================================= + + +class TestEnsureLibreNMSIdCustomField: + """Test _ensure_librenms_id_custom_field signal handler.""" + + def setup_method(self): + """Reset the _executed flag before each test for consistent isolation.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + _ensure_librenms_id_custom_field._executed = False + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_creates_custom_field_when_missing( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """Custom field is created with correct defaults when it does not exist.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [] + MockCustomField.objects.get_or_create.return_value = (mock_cf, True) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + with patch("logging.getLogger") as mock_get_logger: + _ensure_librenms_id_custom_field(sender=None) + + MockCustomField.objects.get_or_create.assert_called_once_with( + name="librenms_id", + defaults={ + "type": "json", + "label": "LibreNMS ID", + "description": "LibreNMS Device ID for synchronization (auto-created by plugin)", + "required": False, + "ui_visible": "if-set", + "ui_editable": "yes", + "is_cloneable": False, + }, + ) + + # Should have added content types for all 4 models + assert mock_cf.object_types.add.call_count == 4 + + # Should log when created + mock_get_logger.assert_called_with("netbox_librenms_plugin") + mock_get_logger.return_value.info.assert_called_once() + + def test_skips_when_already_executed(self): + """Handler is a no-op on second invocation (per-migrate dedup).""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + _ensure_librenms_id_custom_field._executed = True + + with patch("extras.models.CustomField") as MockCustomField: + _ensure_librenms_id_custom_field(sender=None) + MockCustomField.objects.get_or_create.assert_not_called() + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_existing_field_not_recreated( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """When custom field already exists, it is not recreated but types are checked.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2, 3, 4] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + _ensure_librenms_id_custom_field(sender=None) + + # All pks already present, no types should be added + mock_cf.object_types.add.assert_not_called() + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_adds_missing_content_types( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """When some content types are missing, only those are added.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + ct_existing = MagicMock() + ct_existing.pk = 1 + ct_new = MagicMock() + ct_new.pk = 99 + MockContentType.objects.get_for_model.side_effect = [ct_existing, ct_existing, ct_new, ct_new] + + _ensure_librenms_id_custom_field(sender=None) + + assert mock_cf.object_types.add.call_count == 2 + mock_cf.object_types.add.assert_any_call(ct_new) + + @patch("extras.models.CustomField") + def test_exception_does_not_propagate(self, MockCustomField): + """Exceptions during custom field creation are caught and logged.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + MockCustomField.objects.get_or_create.side_effect = Exception("DB not ready") + + with patch("logging.getLogger") as mock_get_logger: + # Should not raise + _ensure_librenms_id_custom_field(sender=None) + + # Verify the exception was logged + logger_instance = mock_get_logger.return_value + logger_instance.exception.assert_called_once() + call_args = logger_instance.exception.call_args + assert "librenms_id" in call_args[0][0] + + # On failure, _executed must NOT be set — failed attempts should allow retry + assert not getattr(_ensure_librenms_id_custom_field, "_executed", False) + + @patch("dcim.models.Interface", new_callable=MagicMock) + @patch("dcim.models.Device", new_callable=MagicMock) + @patch("virtualization.models.VMInterface", new_callable=MagicMock) + @patch("virtualization.models.VirtualMachine", new_callable=MagicMock) + @patch("django.contrib.contenttypes.models.ContentType") + @patch("extras.models.CustomField") + def test_no_log_when_field_already_exists( + self, MockCustomField, MockContentType, mock_vm, mock_vmif, mock_device, mock_iface + ): + """No log message when the custom field already existed.""" + from netbox_librenms_plugin import _ensure_librenms_id_custom_field + + mock_cf = MagicMock() + mock_cf.object_types.values_list.return_value = [1, 2, 3, 4] + MockCustomField.objects.get_or_create.return_value = (mock_cf, False) + + mock_ct = MagicMock() + mock_ct.pk = 1 + MockContentType.objects.get_for_model.return_value = mock_ct + + with patch("logging.getLogger") as mock_get_logger: + _ensure_librenms_id_custom_field(sender=None) + # When the field already exists (created=False), the info log should + # not be emitted. We verify via the logger instance rather than + # asserting getLogger was never called, which is fragile. + logger_instance = mock_get_logger.return_value + logger_instance.info.assert_not_called() diff --git a/netbox_librenms_plugin/tests/test_utils.py b/netbox_librenms_plugin/tests/test_utils.py index 96065ab760..d98ca28b9c 100644 --- a/netbox_librenms_plugin/tests/test_utils.py +++ b/netbox_librenms_plugin/tests/test_utils.py @@ -15,9 +15,12 @@ class TestDeviceTypeMatching: """Test device type matching logic.""" + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") @patch("dcim.models.DeviceType") - def test_match_device_type_exact_match_by_part_number(self, mock_device_type): + def test_match_device_type_exact_match_by_part_number(self, mock_device_type, mock_mapping): """Exact part_number string should match.""" + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.side_effect = mock_mapping.DoesNotExist mock_dt = MagicMock(id=1, model="C9300-48P") mock_device_type.objects.get.return_value = mock_dt @@ -29,9 +32,12 @@ def test_match_device_type_exact_match_by_part_number(self, mock_device_type): assert result["device_type"] == mock_dt assert result["match_type"] == "exact" + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") @patch("dcim.models.DeviceType") - def test_match_device_type_exact_match_by_model(self, mock_device_type): + def test_match_device_type_exact_match_by_model(self, mock_device_type, mock_mapping): """Exact model string should match when part_number fails.""" + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.side_effect = mock_mapping.DoesNotExist mock_dt = MagicMock(id=1, model="WS-C3750X-48P") # Part number lookup fails, model lookup succeeds mock_device_type.DoesNotExist = Exception @@ -48,9 +54,12 @@ def test_match_device_type_exact_match_by_model(self, mock_device_type): assert result["device_type"] == mock_dt assert result["match_type"] == "exact" + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") @patch("dcim.models.DeviceType") - def test_match_device_type_not_found(self, mock_device_type): + def test_match_device_type_not_found(self, mock_device_type, mock_mapping): """Returns None when no match found.""" + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.side_effect = mock_mapping.DoesNotExist mock_device_type.DoesNotExist = Exception mock_device_type.objects.get.side_effect = mock_device_type.DoesNotExist @@ -62,6 +71,22 @@ def test_match_device_type_not_found(self, mock_device_type): assert result["device_type"] is None assert result["match_type"] is None + @patch("netbox_librenms_plugin.models.DeviceTypeMapping") + def test_match_device_type_mapping_match(self, mock_mapping): + """DeviceTypeMapping entry should be used before part_number/model fallback.""" + mock_dt = MagicMock(id=1, model="MX480") + mock_mapping_obj = MagicMock(netbox_device_type=mock_dt) + mock_mapping.DoesNotExist = Exception + mock_mapping.objects.get.return_value = mock_mapping_obj + + from netbox_librenms_plugin.utils import match_librenms_hardware_to_device_type + + result = match_librenms_hardware_to_device_type("Juniper MX480 Internet Backbone Router") + + assert result["matched"] is True + assert result["device_type"] == mock_dt + assert result["match_type"] == "mapping" + def test_match_device_type_empty_hardware(self): """Empty string returns None.""" from netbox_librenms_plugin.utils import match_librenms_hardware_to_device_type diff --git a/netbox_librenms_plugin/urls.py b/netbox_librenms_plugin/urls.py index 63b4e9b0db..ace6126ebe 100644 --- a/netbox_librenms_plugin/urls.py +++ b/netbox_librenms_plugin/urls.py @@ -1,6 +1,6 @@ from django.urls import include, path -from .models import InterfaceTypeMapping +from .models import DeviceTypeMapping, InterfaceTypeMapping, ModuleBayMapping, ModuleTypeMapping, NormalizationRule from .views import ( AddDeviceToLibreNMSView, AssignVCSerialView, @@ -14,12 +14,23 @@ DeviceInterfaceTableView, DeviceIPAddressTableView, DeviceLibreNMSSyncView, + DeviceModuleTableView, DeviceRackUpdateView, DeviceRoleUpdateView, DeviceStatusListView, + DeviceTypeMappingBulkDeleteView, + DeviceTypeMappingBulkImportView, + DeviceTypeMappingChangeLogView, + DeviceTypeMappingCreateView, + DeviceTypeMappingDeleteView, + DeviceTypeMappingEditView, + DeviceTypeMappingListView, + DeviceTypeMappingView, DeviceValidationDetailsView, DeviceVCDetailsView, DeviceVLANTableView, + InstallBranchView, + InstallModuleView, InterfaceTypeMappingBulkDeleteView, InterfaceTypeMappingBulkImportView, InterfaceTypeMappingChangeLogView, @@ -30,6 +41,30 @@ InterfaceTypeMappingView, LibreNMSImportView, LibreNMSSettingsView, + ModuleBayMappingBulkDeleteView, + ModuleBayMappingBulkImportView, + ModuleBayMappingChangeLogView, + ModuleBayMappingCreateView, + ModuleBayMappingDeleteView, + ModuleBayMappingEditView, + ModuleBayMappingListView, + ModuleBayMappingView, + ModuleTypeMappingBulkDeleteView, + ModuleTypeMappingBulkImportView, + ModuleTypeMappingChangeLogView, + ModuleTypeMappingCreateView, + ModuleTypeMappingDeleteView, + ModuleTypeMappingEditView, + ModuleTypeMappingListView, + ModuleTypeMappingView, + NormalizationRuleBulkDeleteView, + NormalizationRuleBulkImportView, + NormalizationRuleChangeLogView, + NormalizationRuleCreateView, + NormalizationRuleDeleteView, + NormalizationRuleEditView, + NormalizationRuleListView, + NormalizationRuleView, RemoveServerMappingView, SaveUserPrefView, SingleCableVerifyView, @@ -72,6 +107,21 @@ DeviceCableTableView.as_view(), name="device_cable_sync", ), + path( + "devices//module-sync/", + DeviceModuleTableView.as_view(), + name="device_module_sync", + ), + path( + "devices//install-module/", + InstallModuleView.as_view(), + name="install_module", + ), + path( + "devices//install-branch/", + InstallBranchView.as_view(), + name="install_branch", + ), path( "devices//ipaddress-sync/", DeviceIPAddressTableView.as_view(), @@ -341,5 +391,173 @@ InterfaceTypeMappingBulkDeleteView.as_view(), name="interfacetypemapping_bulk_delete", ), + # Device type mapping URLs + path( + "device-type-mappings/", + DeviceTypeMappingListView.as_view(), + name="devicetypemapping_list", + ), + path( + "device-type-mappings//", + DeviceTypeMappingView.as_view(), + name="devicetypemapping_detail", + ), + path( + "device-type-mappings/add/", + DeviceTypeMappingCreateView.as_view(), + name="devicetypemapping_add", + ), + path( + "device-type-mappings/import/", + DeviceTypeMappingBulkImportView.as_view(), + name="devicetypemapping_bulk_import", + ), + path( + "device-type-mappings//delete/", + DeviceTypeMappingDeleteView.as_view(), + name="devicetypemapping_delete", + ), + path( + "device-type-mappings//edit/", + DeviceTypeMappingEditView.as_view(), + name="devicetypemapping_edit", + ), + path( + "device-type-mappings//changelog/", + DeviceTypeMappingChangeLogView.as_view(), + name="devicetypemapping_changelog", + kwargs={"model": DeviceTypeMapping}, + ), + path( + "device-type-mappings/delete/", + DeviceTypeMappingBulkDeleteView.as_view(), + name="devicetypemapping_bulk_delete", + ), + # Module type mapping URLs + path( + "module-type-mappings/", + ModuleTypeMappingListView.as_view(), + name="moduletypemapping_list", + ), + path( + "module-type-mappings//", + ModuleTypeMappingView.as_view(), + name="moduletypemapping_detail", + ), + path( + "module-type-mappings/add/", + ModuleTypeMappingCreateView.as_view(), + name="moduletypemapping_add", + ), + path( + "module-type-mappings/import/", + ModuleTypeMappingBulkImportView.as_view(), + name="moduletypemapping_bulk_import", + ), + path( + "module-type-mappings//delete/", + ModuleTypeMappingDeleteView.as_view(), + name="moduletypemapping_delete", + ), + path( + "module-type-mappings//edit/", + ModuleTypeMappingEditView.as_view(), + name="moduletypemapping_edit", + ), + path( + "module-type-mappings//changelog/", + ModuleTypeMappingChangeLogView.as_view(), + name="moduletypemapping_changelog", + kwargs={"model": ModuleTypeMapping}, + ), + path( + "module-type-mappings/delete/", + ModuleTypeMappingBulkDeleteView.as_view(), + name="moduletypemapping_bulk_delete", + ), + # Module Bay Mapping URLs + path( + "module-bay-mappings/", + ModuleBayMappingListView.as_view(), + name="modulebaymapping_list", + ), + path( + "module-bay-mappings//", + ModuleBayMappingView.as_view(), + name="modulebaymapping_detail", + ), + path( + "module-bay-mappings/add/", + ModuleBayMappingCreateView.as_view(), + name="modulebaymapping_add", + ), + path( + "module-bay-mappings/import/", + ModuleBayMappingBulkImportView.as_view(), + name="modulebaymapping_bulk_import", + ), + path( + "module-bay-mappings//delete/", + ModuleBayMappingDeleteView.as_view(), + name="modulebaymapping_delete", + ), + path( + "module-bay-mappings//edit/", + ModuleBayMappingEditView.as_view(), + name="modulebaymapping_edit", + ), + path( + "module-bay-mappings//changelog/", + ModuleBayMappingChangeLogView.as_view(), + name="modulebaymapping_changelog", + kwargs={"model": ModuleBayMapping}, + ), + path( + "module-bay-mappings/delete/", + ModuleBayMappingBulkDeleteView.as_view(), + name="modulebaymapping_bulk_delete", + ), + # Normalization Rule URLs + path( + "normalization-rules/", + NormalizationRuleListView.as_view(), + name="normalizationrule_list", + ), + path( + "normalization-rules//", + NormalizationRuleView.as_view(), + name="normalizationrule_detail", + ), + path( + "normalization-rules/add/", + NormalizationRuleCreateView.as_view(), + name="normalizationrule_add", + ), + path( + "normalization-rules/import/", + NormalizationRuleBulkImportView.as_view(), + name="normalizationrule_bulk_import", + ), + path( + "normalization-rules//delete/", + NormalizationRuleDeleteView.as_view(), + name="normalizationrule_delete", + ), + path( + "normalization-rules//edit/", + NormalizationRuleEditView.as_view(), + name="normalizationrule_edit", + ), + path( + "normalization-rules//changelog/", + NormalizationRuleChangeLogView.as_view(), + name="normalizationrule_changelog", + kwargs={"model": NormalizationRule}, + ), + path( + "normalization-rules/delete/", + NormalizationRuleBulkDeleteView.as_view(), + name="normalizationrule_bulk_delete", + ), path("api/", include("netbox_librenms_plugin.api.urls")), ] diff --git a/netbox_librenms_plugin/utils.py b/netbox_librenms_plugin/utils.py index a5c73dd807..c92dd66727 100644 --- a/netbox_librenms_plugin/utils.py +++ b/netbox_librenms_plugin/utils.py @@ -3,8 +3,8 @@ from typing import Optional from dcim.models import Device -from django.core.exceptions import ObjectDoesNotExist from django.db.models import Q +from django.core.exceptions import ObjectDoesNotExist from django.http import HttpRequest from netbox.config import get_config from netbox.plugins import get_plugin_config @@ -197,7 +197,8 @@ def match_librenms_hardware_to_device_type(hardware_name: str) -> dict: """ Match LibreNMS hardware string to a NetBox DeviceType. - Only performs exact matching on part_number and model fields (case-insensitive). + Checks DeviceTypeMapping table first, then falls back to exact matching + on part_number and model fields (case-insensitive). Args: hardware_name (str): Hardware string from LibreNMS API (e.g., 'C9200L-48P-4X') @@ -206,13 +207,29 @@ def match_librenms_hardware_to_device_type(hardware_name: str) -> dict: dict: Dictionary containing: - matched (bool): Whether a match was found - device_type (DeviceType|None): The matched DeviceType object - - match_type (str|None): Always 'exact' if found, None otherwise + - match_type (str|None): 'mapping' if via DeviceTypeMapping, 'exact' if via + part_number/model, None otherwise """ from dcim.models import DeviceType + from netbox_librenms_plugin.models import DeviceTypeMapping + if not hardware_name or hardware_name == "-": return {"matched": False, "device_type": None, "match_type": None} + # Check DeviceTypeMapping table first + try: + mapping = DeviceTypeMapping.objects.get(librenms_hardware__iexact=hardware_name) + return { + "matched": True, + "device_type": mapping.netbox_device_type, + "match_type": "mapping", + } + except DeviceTypeMapping.DoesNotExist: + pass + except DeviceTypeMapping.MultipleObjectsReturned: + pass + # Try part number exact match try: device_type = DeviceType.objects.get(part_number__iexact=hardware_name) @@ -557,3 +574,123 @@ def migrate_legacy_librenms_id(obj, server_key: str = "default") -> bool: obj, ) return True + + +# Minimum NetBox version that supports {module_path} token in module templates + + +def supports_module_path(): + """Check if the running NetBox supports the {module_path} template token. + + Detects by checking for MODULE_PATH_TOKEN in dcim.constants rather than + comparing version strings — works with patched/pre-release builds too. + """ + try: + from dcim.constants import MODULE_PATH_TOKEN # noqa: F401 + + return True + except ImportError: + return False + + +def module_type_uses_module_path(module_type): + """Check if a ModuleType has any interface templates using {module_path}.""" + return any("{module_path}" in t.name for t in module_type.interfacetemplates.all()) + + +def module_type_uses_module_token(module_type) -> bool: + """Check if a ModuleType has interface templates using the {module} token.""" + try: + from dcim.constants import MODULE_TOKEN + except ImportError: + return False + return any(MODULE_TOKEN in t.name for t in module_type.interfacetemplates.all()) + + +def module_type_is_end_module(module_type) -> bool: + """Return True if this module type defines no module bay templates (i.e., it is a leaf/end module).""" + return not module_type.modulebaytemplates.exists() + + +def has_nested_name_conflict(module_type, module_bay): + """Check if installing this module type in a nested bay would cause a name conflict. + + Returns True when ALL of the following are true: + - The module type has interface templates using only ``{module}`` (not ``{module_path}``) + - The bay is nested (its parent is owned by an installed module) + - There is at least one sibling bay under the same parent + + In this situation NetBox's ``resolve_name()`` replaces ``{module}`` with the + root ancestor's bay position, producing the same interface name for every + sibling at this nesting level. + """ + from dcim.constants import MODULE_TOKEN + + if not module_bay or not module_bay.module_id: + return False # Top-level bay — no conflict + + templates = list(module_type.interfacetemplates.all()) + if not templates: + return False # No interface templates + + uses_module_token = any(MODULE_TOKEN in t.name for t in templates) + if not uses_module_token: + return False # Template doesn't use {module} + + # Count how many unique interface names this template would produce across siblings + # If all siblings resolve to the same name, there's a conflict + from dcim.models import ModuleBay as ModuleBayModel + + sibling_count = ModuleBayModel.objects.filter( + device=module_bay.device, + module_id=module_bay.module_id, + ).count() + + return sibling_count > 1 + + +def apply_normalization_rules(value: str, scope: str, manufacturer=None) -> str: + """Apply NormalizationRule chain to transform a string before matching. + + Rules for the given scope are applied in priority order. Each rule's + regex substitution transforms the output of the previous rule, forming + a pipeline. If no rules match, the original value is returned unchanged. + + When *manufacturer* is given, manufacturer-scoped rules run first, + followed by unscoped (manufacturer=NULL) rules. When *manufacturer* + is ``None``, all rules for the scope run in priority order. + + Args: + value: The raw string to normalize (e.g. '3HE16474AARA01'). + scope: One of NormalizationRule.SCOPE_* constants. + manufacturer: Optional Manufacturer instance to scope rules. + + Returns: + The normalized string after all matching rules have been applied. + """ + from netbox_librenms_plugin.models import NormalizationRule + + if not value: + return value + + if manufacturer: + # Manufacturer-specific rules first, then unscoped rules + for mfg_filter in [{"manufacturer": manufacturer}, {"manufacturer__isnull": True}]: + rules = NormalizationRule.objects.filter(scope=scope, **mfg_filter).order_by("priority", "pk") + for rule in rules: + try: + value = re.sub(rule.match_pattern, rule.replacement, value) + except re.error: + logger.error( + "Invalid regex in NormalizationRule pk=%s pattern=%r — skipping", rule.pk, rule.match_pattern + ) + else: + rules = NormalizationRule.objects.filter(scope=scope).order_by("priority", "pk") + for rule in rules: + try: + value = re.sub(rule.match_pattern, rule.replacement, value) + except re.error: + logger.error( + "Invalid regex in NormalizationRule pk=%s pattern=%r — skipping", rule.pk, rule.match_pattern + ) + return value diff --git a/netbox_librenms_plugin/views/__init__.py b/netbox_librenms_plugin/views/__init__.py index f9c3db7790..bb27fde9f3 100644 --- a/netbox_librenms_plugin/views/__init__.py +++ b/netbox_librenms_plugin/views/__init__.py @@ -10,6 +10,7 @@ from .base.interfaces_view import BaseInterfaceTableView # noqa: F401 from .base.ip_addresses_view import BaseIPAddressTableView, SingleIPAddressVerifyView # noqa: F401 from .base.librenms_sync_view import BaseLibreNMSSyncView # noqa: F401 +from .base.modules_view import InstallBranchView, InstallModuleView # noqa: F401 from .base.vlan_table_view import BaseVLANTableView # noqa: F401 from .imports import ( # noqa: F401 BulkImportConfirmView, @@ -24,6 +25,14 @@ SaveUserPrefView, ) from .mapping_views import ( # noqa: F401 + DeviceTypeMappingBulkDeleteView, + DeviceTypeMappingBulkImportView, + DeviceTypeMappingChangeLogView, + DeviceTypeMappingCreateView, + DeviceTypeMappingDeleteView, + DeviceTypeMappingEditView, + DeviceTypeMappingListView, + DeviceTypeMappingView, InterfaceTypeMappingBulkDeleteView, InterfaceTypeMappingBulkImportView, InterfaceTypeMappingChangeLogView, @@ -32,12 +41,37 @@ InterfaceTypeMappingEditView, InterfaceTypeMappingListView, InterfaceTypeMappingView, + ModuleBayMappingBulkDeleteView, + ModuleBayMappingBulkImportView, + ModuleBayMappingChangeLogView, + ModuleBayMappingCreateView, + ModuleBayMappingDeleteView, + ModuleBayMappingEditView, + ModuleBayMappingListView, + ModuleBayMappingView, + ModuleTypeMappingBulkDeleteView, + ModuleTypeMappingBulkImportView, + ModuleTypeMappingChangeLogView, + ModuleTypeMappingCreateView, + ModuleTypeMappingDeleteView, + ModuleTypeMappingEditView, + ModuleTypeMappingListView, + ModuleTypeMappingView, + NormalizationRuleBulkDeleteView, + NormalizationRuleBulkImportView, + NormalizationRuleChangeLogView, + NormalizationRuleCreateView, + NormalizationRuleDeleteView, + NormalizationRuleEditView, + NormalizationRuleListView, + NormalizationRuleView, ) from .object_sync import ( # noqa: F401 DeviceCableTableView, DeviceInterfaceTableView, DeviceIPAddressTableView, DeviceLibreNMSSyncView, + DeviceModuleTableView, DeviceVLANTableView, SaveVlanGroupOverridesView, SingleInterfaceVerifyView, diff --git a/netbox_librenms_plugin/views/base/cables_view.py b/netbox_librenms_plugin/views/base/cables_view.py index 4cbcdb7a36..46fd0f21b2 100644 --- a/netbox_librenms_plugin/views/base/cables_view.py +++ b/netbox_librenms_plugin/views/base/cables_view.py @@ -25,7 +25,6 @@ class BaseCableTableView(LibreNMSPermissionMixin, LibreNMSAPIMixin, CacheMixin, model = None # To be defined in subclasses partial_template_name = "netbox_librenms_plugin/_cable_sync_content.html" - interface_name_field = get_interface_name_field() def get_object(self, pk): """Retrieve the object (Device or VirtualMachine).""" @@ -54,11 +53,17 @@ def get_links_data(self, obj): if not success or "error" in data: return None + interface_name_field = get_interface_name_field(getattr(self, "request", None)) ports_data = self.get_ports_data(obj) local_ports_map = {} for port in ports_data.get("ports", []): - port_id = str(port["port_id"]) - port_name = port[self.interface_name_field] + raw_port_id = port.get("port_id") + if raw_port_id is None: + continue + port_id = str(raw_port_id) + port_name = port.get(interface_name_field) + if port_name is None: + continue local_ports_map[port_id] = port_name links = data.get("links", []) diff --git a/netbox_librenms_plugin/views/base/librenms_sync_view.py b/netbox_librenms_plugin/views/base/librenms_sync_view.py index 196c744907..c723859904 100644 --- a/netbox_librenms_plugin/views/base/librenms_sync_view.py +++ b/netbox_librenms_plugin/views/base/librenms_sync_view.py @@ -87,6 +87,7 @@ def get_context_data(self, request, obj): cable_context = self.get_cable_context(request, obj) ip_context = self.get_ip_context(request, obj) vlan_context = self.get_vlan_context(request, obj) + module_context = self.get_module_context(request, obj) interface_name_field = get_interface_name_field(request) @@ -104,6 +105,7 @@ def get_context_data(self, request, obj): "cable_sync": cable_context, "ip_sync": ip_context, "vlan_sync": vlan_context, + "module_sync": module_context, "v1v2form": AddToLIbreSNMPV1V2(prefix="v1v2"), "v3form": AddToLIbreSNMPV3(prefix="v3"), "librenms_device_id": self.librenms_id, @@ -280,6 +282,7 @@ def get_librenms_device_info(self, obj): if netbox_identities & librenms_identities: mismatched_device = False else: + # Device is still found (we have librenms_id), just mismatched mismatched_device = True librenms_device_details["netbox_dns_name"] = netbox_dns_name or "-" @@ -318,6 +321,61 @@ def get_vlan_context(self, request, obj): """ return None + def get_module_context(self, request, obj): + """ + Get the context data for module sync. + Subclasses should override this method if applicable. + """ + return None + + @staticmethod + def _build_all_server_mappings(obj, active_server_key): + """Build a list of all LibreNMS server mappings for the given device. + + Each entry describes one server<->ID mapping stored in the ``librenms_id`` + custom field: + + * ``server_key`` – the key as stored in the CF dict (or ``"legacy"``). + * ``display_name`` – human-readable name from PLUGINS_CONFIG, or the key. + * ``librenms_url`` – base URL of that server (``None`` when not configured). + * ``device_id`` – the integer device ID on that server. + * ``device_url`` – direct URL to the device page on that server (or ``None``). + * ``is_configured``– True when the server key exists in current plugin config. + * ``is_active`` – True when this is the currently active server. + + Returns ``None`` for legacy bare-int format (no per-server info to show) + and ``None`` when the CF is absent/invalid. + """ + cf_value = obj.custom_field_data.get("librenms_id") + if not isinstance(cf_value, dict) or not cf_value: + return None + + plugins_cfg = django_settings.PLUGINS_CONFIG.get("netbox_librenms_plugin", {}) + servers_config = plugins_cfg.get("servers", {}) + + result = [] + for sk, did in cf_value.items(): + srv_cfg = servers_config.get(sk) + is_configured = srv_cfg is not None + librenms_url = srv_cfg.get("librenms_url") if srv_cfg else None + display_name = (srv_cfg.get("display_name") or sk) if srv_cfg else sk + device_url = f"{librenms_url}/device/device={did}/" if librenms_url else None + result.append( + { + "server_key": sk, + "display_name": display_name, + "librenms_url": librenms_url, + "device_id": did, + "device_url": device_url, + "is_configured": is_configured, + "is_active": sk == active_server_key, + } + ) + + # Sort: active first, then configured, then orphaned + result.sort(key=lambda e: 0 if e["is_active"] else (1 if e["is_configured"] else 2)) + return result or None + @staticmethod def _strip_vc_pattern(name): """Strip the VC member naming suffix from a device name. diff --git a/netbox_librenms_plugin/views/base/modules_view.py b/netbox_librenms_plugin/views/base/modules_view.py new file mode 100644 index 0000000000..68a94b2fca --- /dev/null +++ b/netbox_librenms_plugin/views/base/modules_view.py @@ -0,0 +1,1064 @@ +from django.contrib import messages +from django.core.cache import cache +from django.db import transaction +from django.shortcuts import get_object_or_404, redirect, render +from django.urls import reverse +from django.utils import timezone +from django.views import View + +from netbox_librenms_plugin.views.mixins import ( + CacheMixin, + LibreNMSAPIMixin, + LibreNMSPermissionMixin, + NetBoxObjectPermissionMixin, +) + + +# entPhysicalClass values relevant for module sync +# Includes vendor-specific classes (Nokia TIMETRA-CHASSIS-MIB uses ioModule, cpmModule, etc.) +INVENTORY_CLASSES = { + "module", + "powerSupply", + "fan", + "port", + "container", + "ioModule", + "cpmModule", + "mdaModule", + "fabricModule", + "xioModule", +} + +# Model name values that indicate a generic/empty container (not real hardware) +_GENERIC_CONTAINER_MODELS = {"", "BUILTIN", "Default", "N/A"} + + +class BaseModuleTableView(LibreNMSPermissionMixin, LibreNMSAPIMixin, CacheMixin, View): + """ + Base view for synchronizing module/inventory data from LibreNMS. + Fetches inventory, matches against NetBox module bays and module types, + and renders a comparison table. + """ + + model = None + partial_template_name = "netbox_librenms_plugin/_module_sync_content.html" + + def get_object(self, pk): + """Retrieve the object (Device).""" + return get_object_or_404(self.model, pk=pk) + + def get_table(self, data, obj): + """Returns the table class. Subclasses should override.""" + raise NotImplementedError("Subclasses must implement get_table()") + + def post(self, request, pk): + """Fetch inventory from LibreNMS, cache it, and render the module sync table.""" + obj = self.get_object(pk) + + self.librenms_id = self.librenms_api.get_librenms_id(obj) + if not self.librenms_id: + messages.error(request, "Device not found in LibreNMS.") + return render( + request, + self.partial_template_name, + {"module_sync": {"object": obj, "table": None, "cache_expiry": None}}, + ) + + success, inventory_data = self.librenms_api.get_device_inventory(self.librenms_id) + + if not success: + messages.error(request, f"Failed to fetch inventory from LibreNMS: {inventory_data}") + return render( + request, + self.partial_template_name, + {"module_sync": {"object": obj, "table": None, "cache_expiry": None}}, + ) + + # Fetch transceiver data and merge with inventory + inventory_data = self._merge_transceiver_data(inventory_data) + + # Cache the merged inventory data + cache.set( + self.get_cache_key(obj, "inventory"), + inventory_data, + timeout=self.librenms_api.cache_timeout, + ) + + context = self._build_context(request, obj, inventory_data) + messages.success(request, "Inventory data refreshed successfully.") + return render(request, self.partial_template_name, {"module_sync": context}) + + def get_context_data(self, request, obj): + """Get context from cache (used by the main sync view on initial page load).""" + cached_data = cache.get(self.get_cache_key(obj, "inventory")) + if not cached_data: + return {"table": None, "object": obj, "cache_expiry": None} + return self._build_context(request, obj, cached_data) + + def _build_context(self, request, obj, inventory_data): + """Build context with matched inventory items and table.""" + # Build a lookup of all inventory items by index for parent resolution + index_map = {item["entPhysicalIndex"]: item for item in inventory_data} + + # Store manufacturer for normalization rules in _build_row + self._device_manufacturer = getattr(getattr(obj, "device_type", None), "manufacturer", None) + + # Get NetBox module bays and modules for this device + device_bays, module_scoped_bays = self._get_module_bays(obj) + module_types = self._get_module_types() + + # Collect top-level items and their sub-components + # Include synthetic transceiver items (from vendors without ENTITY-MIB SFP data) + # Exclude items that have any ancestor with an INVENTORY_CLASSES class + # (they appear as sub-components under that ancestor) + top_items = [] + for item in inventory_data: + if item.get("_from_transceiver_api"): + top_items.append(item) + continue + phys_class = item.get("entPhysicalClass") + if phys_class not in INVENTORY_CLASSES: + continue + # Skip items with generic model names (not real hardware). + # Containers with empty model are physical slot representations. + model = (item.get("entPhysicalModelName") or "").strip() + if phys_class == "container" and model in _GENERIC_CONTAINER_MODELS: + continue + if model and model in _GENERIC_CONTAINER_MODELS: + continue + # Walk up ancestor chain; skip if any ancestor is an inventory-class item. + # Containers with empty model are physical slot/bay representations, not + # real modules — skip them so children can be top-level items. + is_descendant = False + current_idx = item.get("entPhysicalContainedIn", 0) + for _ in range(10): + if not current_idx or current_idx not in index_map: + break + ancestor = index_map[current_idx] + anc_class = ancestor.get("entPhysicalClass") + if anc_class in INVENTORY_CLASSES: + anc_model = (ancestor.get("entPhysicalModelName") or "").strip() + # Empty-model containers are just physical slot representations + if anc_class == "container" and not anc_model: + current_idx = ancestor.get("entPhysicalContainedIn", 0) + continue + is_descendant = True + break + current_idx = ancestor.get("entPhysicalContainedIn", 0) + if is_descendant: + continue + top_items.append(item) + + table_data = [] + from netbox_librenms_plugin.utils import apply_normalization_rules + + # Build combined bay lookup so synthetic transceiver entries (which may + # live inside installed modules) can find their module-scoped bays. + all_bays = dict(device_bays) + for scope_bays in module_scoped_bays.values(): + all_bays.update(scope_bays) + + for item in top_items: + # Transceiver API entries may live inside installed modules, so they + # need the full bay map. ENTITY-MIB top-level items must only match + # device-level bays to avoid name collisions with module-scoped bays + # that share the same name as a device bay. + item_bays = all_bays if item.get("_from_transceiver_api") else device_bays + row = self._build_row(item, index_map, item_bays, module_types, depth=0) + parent_idx = len(table_data) + table_data.append(row) + + # Determine which bays sub-components should match against: + # If parent matched a bay with an installed module, use that module's child bays. + # If parent matched a bay but it's NOT installed, children can't be installed + # individually (parent must be installed first to create child bays). + parent_module_id = None + parent_bay_matched_but_uninstalled = False + if row.get("module_bay_id"): + matched_bay = item_bays.get(row["module_bay"]) + if matched_bay and hasattr(matched_bay, "installed_module") and matched_bay.installed_module: + parent_module_id = matched_bay.installed_module.pk + else: + # Parent matched a bay but it's not installed yet + parent_bay_matched_but_uninstalled = True + + if parent_bay_matched_but_uninstalled: + # Empty dict: children can't match any bay individually + child_bays = {} + elif parent_module_id: + child_bays = module_scoped_bays.get(parent_module_id, {}) + else: + child_bays = device_bays + + # Find sub-components with a model name (transceivers, converters, etc.) + # Track bay scope per depth level so nested modules use correct bays + bays_by_depth = {0: child_bays} + sub_items = self._get_sub_components(item["entPhysicalIndex"], inventory_data) + for depth, sub_item in sub_items: + scope_bays = bays_by_depth.get(depth, child_bays) + sub_row = self._build_row(sub_item, index_map, scope_bays, module_types, depth=depth) + table_data.append(sub_row) + + # If this sub-item matched an installed module, deeper items use its bays + if sub_row.get("module_bay_id"): + matched_sub_bay = scope_bays.get(sub_row["module_bay"]) + if ( + matched_sub_bay + and hasattr(matched_sub_bay, "installed_module") + and matched_sub_bay.installed_module + ): + sub_module_id = matched_sub_bay.installed_module.pk + bays_by_depth[depth + 1] = module_scoped_bays.get(sub_module_id, {}) + + # Mark parent if any child is installable + if sub_row.get("can_install"): + table_data[parent_idx]["has_installable_children"] = True + + # When parent is installable but children can't match bays yet + # (parent module not installed), enable "Install Branch" if any child + # has a matching module type (branch install handles bay creation). + if ( + parent_bay_matched_but_uninstalled + and row.get("can_install") + and not table_data[parent_idx].get("has_installable_children") + ): + for _depth, sub_item in sub_items: + sub_model = (sub_item.get("entPhysicalModelName") or "").strip() + if not sub_model: + continue + matched = module_types.get(sub_model) + if not matched: + normalized = apply_normalization_rules( + sub_model, + "module_type", + manufacturer=getattr(self, "_device_manufacturer", None), + ) + matched = module_types.get(normalized) + if matched: + table_data[parent_idx]["has_installable_children"] = True + break + + # Sort top-level groups by status, keeping children after their parent + table_data = self._sort_with_hierarchy(table_data) + + table = self.get_table(table_data, obj) + table.configure(request) + + cache_ttl = getattr(cache, "ttl", lambda k: None)(self.get_cache_key(obj, "inventory")) + cache_expiry = timezone.now() + timezone.timedelta(seconds=cache_ttl) if cache_ttl is not None else None + + return { + "table": table, + "object": obj, + "cache_expiry": cache_expiry, + } + + def _merge_transceiver_data(self, inventory_data): + """Merge transceiver API data with entity inventory. + + For vendors like Nokia that don't expose SFPs in ENTITY-MIB, + the transceiver API provides SFP model, serial, and type info. + + Strategy: + - For transceivers matching existing inventory items by entity_physical_index: + supplement entPhysicalModelName if empty + - For transceivers NOT in inventory: create synthetic inventory items + so they appear in the modules table + """ + success, transceivers = self.librenms_api.get_device_transceivers(self.librenms_id) + if not success or not transceivers: + return inventory_data + + # Build lookup of existing inventory items by index and serial + inv_by_index = {item["entPhysicalIndex"]: item for item in inventory_data} + inv_serials = { + (item.get("entPhysicalSerialNum") or "").strip() + for item in inventory_data + if (item.get("entPhysicalSerialNum") or "").strip() + } + + # Build port_id → ifName lookup for better synthetic item naming + port_name_map = self._build_port_name_map(transceivers) + + # Types that are containers, not real transceiver modules + SKIP_TYPES = {"Port Container", "Port", ""} + + for txr in transceivers: + ent_idx = txr.get("entity_physical_index") + if not ent_idx: + continue + + model = (txr.get("model") or "").strip() + serial = (txr.get("serial") or "").strip() + txr_type = (txr.get("type") or "").strip() + + # Skip containers and entries with no useful data + if txr_type in SKIP_TYPES and not model and not serial: + continue + + # Use transceiver type as model fallback (e.g., "CFP2/QSFP28") + display_model = model or (txr_type if txr_type not in SKIP_TYPES else "") + + if ent_idx in inv_by_index: + # Supplement existing inventory item if model is missing + existing = inv_by_index[ent_idx] + if not (existing.get("entPhysicalModelName") or "").strip() and display_model: + existing["entPhysicalModelName"] = display_model + if not (existing.get("entPhysicalSerialNum") or "").strip() and serial: + existing["entPhysicalSerialNum"] = serial + else: + # Skip if serial already exists in ENTITY-MIB data (avoid duplicates) + if serial and serial in inv_serials: + continue + # Create synthetic inventory item for SFPs not in entity inventory + port_id = txr.get("port_id", 0) + ifname = port_name_map.get(port_id) + if ifname: + name = ifname + elif port_id: + name = f"Transceiver (port {port_id})" + else: + name = f"Transceiver {ent_idx}" + + synthetic = { + "entPhysicalIndex": ent_idx, + "entPhysicalName": name, + "entPhysicalClass": "port", + "entPhysicalModelName": display_model, + "entPhysicalSerialNum": serial, + "entPhysicalDescr": txr_type, + "entPhysicalContainedIn": 0, + "_from_transceiver_api": True, + } + inventory_data.append(synthetic) + + return inventory_data + + def _build_port_name_map(self, transceivers): + """Build port_id → ifName mapping for transceiver ports. + + Fetches port data from LibreNMS to resolve port IDs to interface names, + enabling better bay matching for synthetic transceiver items (e.g., + Nokia 1/1/c1 instead of opaque port IDs). + """ + port_ids = {txr.get("port_id") for txr in transceivers if txr.get("port_id")} + if not port_ids: + return {} + + success, ports_data = self.librenms_api.get_ports(self.librenms_id) + if not success or not isinstance(ports_data, dict): + return {} + + return { + p["port_id"]: p["ifName"] + for p in ports_data.get("ports", []) + if p.get("port_id") in port_ids and p.get("ifName") + } + + def _get_sub_components(self, parent_idx, inventory_data): + """Find descendant items with a model name (real hardware, not empty containers). + + Returns list of (depth, item) tuples. + """ + results = [] + self._collect_descendants(parent_idx, inventory_data, depth=1, results=results) + return results + + def _collect_descendants(self, parent_idx, inventory_data, depth, results): + """Recursively collect descendant items that have a model name.""" + children = [i for i in inventory_data if i.get("entPhysicalContainedIn") == parent_idx] + for child in children: + model = (child.get("entPhysicalModelName") or "").strip() + if model and model not in _GENERIC_CONTAINER_MODELS: + results.append((depth, child)) + # Continue looking for deeper components (e.g., SFPs inside converters) + self._collect_descendants(child["entPhysicalIndex"], inventory_data, depth + 1, results) + else: + # Skip generic/empty items, but check their children + self._collect_descendants(child["entPhysicalIndex"], inventory_data, depth, results) + + def _sort_with_hierarchy(self, table_data): + """Sort table keeping children grouped under their parent.""" + status_order = {"Installed": 0, "Serial Mismatch": 1, "Matched": 2, "No Type": 3, "No Bay": 4, "Unmatched": 5} + + # Group into top-level items with their children + groups = [] + current_group = None + for row in table_data: + if row.get("depth", 0) == 0: + current_group = {"parent": row, "children": []} + groups.append(current_group) + elif current_group is not None: + current_group["children"].append(row) + + # Sort groups by parent status + groups.sort(key=lambda g: status_order.get(g["parent"]["status"], 99)) + + # Flatten back + result = [] + for group in groups: + result.append(group["parent"]) + result.extend(group["children"]) + return result + + def _get_module_bays(self, obj): + """Get module bays for the device, organized by scope. + + Returns: + tuple: (device_bays, module_bays) where: + - device_bays: {name: bay} for device-level bays (module=None) + - module_bays: {module_id: {name: bay}} for bays created by installed modules + """ + from dcim.models import ModuleBay + + bays = ModuleBay.objects.filter(device=obj).select_related("installed_module__module_type") + device_bays = {} + module_scoped_bays = {} + for bay in bays: + if bay.module_id: + module_scoped_bays.setdefault(bay.module_id, {})[bay.name] = bay + else: + device_bays[bay.name] = bay + return device_bays, module_scoped_bays + + def _get_module_types(self): + """Get all module types, indexed by model (part_number), with ModuleTypeMapping checked first.""" + from dcim.models import ModuleType + + from netbox_librenms_plugin.models import ModuleTypeMapping + + # Build base lookup from NetBox module types + types = ModuleType.objects.all().select_related("manufacturer") + result = {} + for mt in types: + result[mt.model] = mt + if mt.part_number and mt.part_number != mt.model: + result[mt.part_number] = mt + + # Overlay with explicit ModuleTypeMapping entries (take priority) + for mapping in ModuleTypeMapping.objects.select_related("netbox_module_type__manufacturer"): + result[mapping.librenms_model] = mapping.netbox_module_type + + return result + + def _find_parent_container_name(self, item, index_map): + """Resolve the parent container name for an inventory item.""" + contained_in = item.get("entPhysicalContainedIn", 0) + if contained_in == 0: + return None + parent = index_map.get(contained_in) + if parent: + return parent.get("entPhysicalName", "") + return None + + def _match_module_bay(self, item, index_map, module_bays): + """ + Try to match an inventory item to a NetBox ModuleBay. + Checks ModuleBayMapping table first (exact then regex), then falls back + to exact parent name match, then positional matching. + """ + import re + + from netbox_librenms_plugin.models import ModuleBayMapping + + parent_name = self._find_parent_container_name(item, index_map) + item_name = item.get("entPhysicalName", "") + item_descr = item.get("entPhysicalDescr", "") + phys_class = item.get("entPhysicalClass", "") + + # Build candidate names: parent, item name, item description + candidate_names = [n for n in [parent_name, item_name, item_descr] if n] + + # Check ModuleBayMapping table for each candidate (exact match) + for name in candidate_names: + filters = {"librenms_name": name, "is_regex": False} + if phys_class: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class=phys_class).first() + if not mapping: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + else: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + + if mapping and mapping.netbox_bay_name in module_bays: + return module_bays[mapping.netbox_bay_name] + + # Regex pattern matching on all candidate names + for name in candidate_names: + bay = self._lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping) + if bay and self._fpc_slot_matches(name, bay): + return bay + + # Fallback: exact match on candidate names against bay dict + for name in candidate_names: + if name in module_bays: + return module_bays[name] + + # Positional fallback: determine slot number from container sibling order + # Handles SFPs inside converters where containers are unnamed + bay = self._match_bay_by_position(item, index_map, module_bays) + if bay: + return bay + + return None + + @staticmethod + def _fpc_slot_matches(candidate_name, bay): + """Validate that a regex-matched bay's parent slot position is consistent with + a positional descriptor like 'Model @ FPC/pic/port'. + + Returns True if the descriptor has no FPC reference, or if the bay's parent + module slot position matches the FPC number in the descriptor. Prevents + orphaned top-level items (e.g. QSFP @ 1/1/1 when FPC1 is not installed) + from incorrectly matching bays belonging to a different FPC's module. + """ + import re as _re + + match = _re.search(r"@\s+(\d+)/", candidate_name) + if not match: + return True + expected_fpc = match.group(1) + module = getattr(bay, "module", None) + if not module: + return True + parent_bay = getattr(module, "module_bay", None) + if not parent_bay: + return True + return parent_bay.position == expected_fpc + + @staticmethod + def _lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping): + """Try regex ModuleBayMapping patterns against a name. + + Returns matched module bay or None. + """ + regex_filters = {"is_regex": True} + if phys_class: + regex_mappings = list(ModuleBayMapping.objects.filter(**regex_filters, librenms_class=phys_class)) + list( + ModuleBayMapping.objects.filter(**regex_filters, librenms_class="") + ) + else: + regex_mappings = list(ModuleBayMapping.objects.filter(**regex_filters, librenms_class="")) + + for mapping in regex_mappings: + try: + match = re.fullmatch(mapping.librenms_name, name) + except re.error: + continue + if match: + resolved_bay = match.expand(mapping.netbox_bay_name) + if resolved_bay in module_bays: + bay = module_bays[resolved_bay] + if BaseModuleTableView._fpc_slot_matches(name, bay): + return bay + return None + + @staticmethod + def _match_bay_by_position(item, index_map, module_bays): + """Match bay by item's positional order among container siblings. + + When an item is inside a container (no model), walk up to find the + nearest ancestor with a model, count which container slot the item + occupies, and match to the bay by number (e.g., SFP 1, SFP 2). + """ + # Walk up through modelless containers to find the parent with a model + current_idx = item.get("entPhysicalContainedIn", 0) + container_idx = None + for _ in range(5): + if not current_idx or current_idx not in index_map: + return None + ancestor = index_map[current_idx] + model = (ancestor.get("entPhysicalModelName") or "").strip() + if model: + # Found the parent with a model; container_idx is the intermediate container + break + container_idx = current_idx + current_idx = ancestor.get("entPhysicalContainedIn", 0) + else: + return None + + if not container_idx: + return None + + # Determine position: count siblings of the container under the parent + parent_with_model_idx = current_idx + siblings = sorted( + [i for i in index_map.values() if i.get("entPhysicalContainedIn") == parent_with_model_idx], + key=lambda x: x.get("entPhysicalParentRelPos", 0), + ) + slot_num = None + for i, sib in enumerate(siblings): + if sib["entPhysicalIndex"] == container_idx: + slot_num = i + 1 + break + + if slot_num is None: + return None + + # Try common bay naming patterns + for pattern in [f"SFP {slot_num}", f"Slot {slot_num}", f"Bay {slot_num}", f"Port {slot_num}"]: + if pattern in module_bays: + return module_bays[pattern] + + return None + + def _build_row(self, item, index_map, module_bays, module_types, depth=0): + """Build a single table row from a LibreNMS inventory item.""" + from netbox_librenms_plugin.utils import ( + apply_normalization_rules, + has_nested_name_conflict, + module_type_is_end_module, + module_type_uses_module_path, + module_type_uses_module_token, + supports_module_path, + ) + + model_name = item.get("entPhysicalModelName", "") or "" + serial = item.get("entPhysicalSerialNum", "") or "" + phys_class = item.get("entPhysicalClass", "") + name = item.get("entPhysicalName", "") or "-" + description = item.get("entPhysicalDescr", "") or "" + + # Match to NetBox module bay + matched_bay = self._match_module_bay(item, index_map, module_bays) + + # Match to NetBox module type (direct lookup, then normalization fallback) + matched_type = module_types.get(model_name) if model_name else None + if not matched_type and model_name: + normalized = apply_normalization_rules( + model_name, "module_type", manufacturer=getattr(self, "_device_manufacturer", None) + ) + if normalized != model_name: + matched_type = module_types.get(normalized) + + # Badge flags — purely informational, never block installation + needs_module_path = matched_type and module_type_uses_module_path(matched_type) + # {module_path} used but NetBox version does not support it → "Upgrade NetBox" hint + netbox_upgrade_needed = bool(needs_module_path and not supports_module_path()) + # End module still using old {module} when {module_path} is available → "Upgrade module-type" hint + suggest_type_upgrade = bool( + matched_type + and supports_module_path() + and module_type_is_end_module(matched_type) + and module_type_uses_module_token(matched_type) + ) + + # Check for nested module naming conflicts + name_conflict = matched_type and matched_bay and has_nested_name_conflict(matched_type, matched_bay) + + # Determine status + status = self._determine_status(matched_bay, matched_type, serial) + + row = { + "name": name, + "model": model_name or "-", + "serial": serial or "-", + "description": description, + "item_class": phys_class, + "module_bay": matched_bay.name if matched_bay else "-", + "module_type": matched_type.model if matched_type else "-", + "status": status, + "row_class": "", + "can_install": False, + "module_bay_id": matched_bay.pk if matched_bay else None, + "module_type_id": matched_type.pk if matched_type else None, + "depth": depth, + "ent_physical_index": item.get("entPhysicalIndex"), + "has_installable_children": False, + } + + if netbox_upgrade_needed: + row["row_class"] = "table-warning" + row["module_path_warning"] = ( + "This module type uses {module_path} in its interface templates. " + "The current NetBox version does not support {module_path} yet — " + "installation will proceed but interface naming may not work as expected. " + "Upgrade NetBox to enable full {module_path} support." + ) + + if suggest_type_upgrade: + row["module_type_upgrade_hint"] = ( + "This module type uses {module} in its interface templates. " + "Since this NetBox version supports {module_path}, consider updating " + "the module type's interface templates to use {module_path} for " + "precise per-slot interface naming." + ) + + if name_conflict: + row["row_class"] = "table-warning" + row["name_conflict_warning"] = ( + "This module type uses {module} in its interface template. " + "Installing multiple siblings will create duplicate interface names. " + "An interface naming plugin with a rewrite rule for this module type can resolve this." + ) + + # Add URLs for matched objects + if matched_bay: + row["module_bay_url"] = matched_bay.get_absolute_url() + # Check if a module is already installed in this bay + if hasattr(matched_bay, "installed_module") and matched_bay.installed_module: + installed = matched_bay.installed_module + row["installed_module"] = installed + row["module_url"] = installed.get_absolute_url() + # Check serial match + if serial and installed.serial and installed.serial.strip() == serial.strip(): + status = "Installed" + row["row_class"] = "table-success" + elif serial and installed.serial and installed.serial.strip() != serial.strip(): + status = "Serial Mismatch" + row["row_class"] = "table-danger" + else: + status = "Installed" + row["row_class"] = "table-success" + row["status"] = status + elif matched_type: + # Bay exists, type matched, no module installed → can install + row["can_install"] = True + + if matched_type: + row["module_type_url"] = matched_type.get_absolute_url() + + return row + + def _determine_status(self, matched_bay, matched_type, serial): + """Determine the sync status for an inventory item.""" + if matched_bay and matched_type: + return "Matched" + if not matched_bay: + return "No Bay" + if not matched_type: + return "No Type" + return "Unmatched" + + +class InstallModuleView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, View): + """Install a NetBox Module into a ModuleBay from LibreNMS inventory data.""" + + def post(self, request, pk): + from dcim.models import Device, Module, ModuleBay, ModuleType + + self.required_object_permissions = {"POST": [("add", Module)]} + if error := self.require_all_permissions_json("POST"): + return error + + device = get_object_or_404(Device, pk=pk) + module_bay_id = request.POST.get("module_bay_id") + module_type_id = request.POST.get("module_type_id") + serial = request.POST.get("serial", "").strip() + + if not module_bay_id or not module_type_id: + messages.error(request, "Missing module bay or module type.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + module_bay = get_object_or_404(ModuleBay, pk=module_bay_id, device=device) + module_type = get_object_or_404(ModuleType, pk=module_type_id) + + # Check if bay already has a module installed + if hasattr(module_bay, "installed_module") and module_bay.installed_module: + messages.warning(request, f"Module bay '{module_bay.name}' already has a module installed.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + try: + with transaction.atomic(): + module = Module( + device=device, + module_bay=module_bay, + module_type=module_type, + serial=serial, + status="active", + ) + module.full_clean() + module.save() + + messages.success( + request, f"Installed {module_type.model} in {module_bay.name} (serial: {serial or 'N/A'})." + ) + except Exception as e: + messages.error(request, f"Failed to install module: {e}") + + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + +class InstallBranchView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, CacheMixin, View): + """Install a module and all its installable descendants from LibreNMS inventory.""" + + def post(self, request, pk): + from dcim.models import Device, Module, ModuleBay, ModuleType + + self.required_object_permissions = {"POST": [("add", Module)]} + if error := self.require_all_permissions_json("POST"): + return error + + device = get_object_or_404(Device, pk=pk) + parent_index = request.POST.get("parent_index") + + if not parent_index: + messages.error(request, "Missing parent inventory index.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + try: + parent_index = int(parent_index) + except ValueError: + messages.error(request, "Invalid parent inventory index.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Get cached inventory data + cached_data = cache.get(self.get_cache_key(device, "inventory")) + if not cached_data: + messages.error(request, "No cached inventory data. Please refresh modules first.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Build index map and collect the branch to install + index_map = {item["entPhysicalIndex"]: item for item in cached_data} + branch_items = self._collect_branch(parent_index, cached_data) + + if not branch_items: + messages.warning(request, "No installable items found in this branch.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Load module types (with mappings) + module_types = self._get_module_types() + + # Install top-down: each install may create new child bays + installed = [] + skipped = [] + failed = [] + + try: + with transaction.atomic(): + for item in branch_items: + result = self._install_single( + device, + item, + index_map, + module_types, + ModuleBay, + ModuleType, + Module, + ) + if result["status"] == "installed": + installed.append(result["name"]) + elif result["status"] == "skipped": + skipped.append(f"{result['name']}: {result['reason']}") + else: + failed.append(f"{result['name']}: {result['reason']}") + except Exception as e: + messages.error(request, f"Branch install failed: {e}") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + # Report results + if installed: + messages.success(request, f"Installed {len(installed)} module(s): {', '.join(installed)}") + if skipped: + messages.info(request, f"Skipped {len(skipped)}: {'; '.join(skipped)}") + if failed: + messages.warning(request, f"Failed {len(failed)}: {'; '.join(failed)}") + + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + def _collect_branch(self, parent_index, inventory_data): + """Collect all items in a branch depth-first, parent first. + + Returns items in install order (parent before children). + """ + items = [] + parent = next((i for i in inventory_data if i["entPhysicalIndex"] == parent_index), None) + if parent: + model = (parent.get("entPhysicalModelName") or "").strip() + if model: + items.append(parent) + self._collect_children(parent_index, inventory_data, items) + return items + + def _collect_children(self, parent_idx, inventory_data, items): + """Recursively collect children with models, depth-first.""" + children = [i for i in inventory_data if i.get("entPhysicalContainedIn") == parent_idx] + for child in children: + model = (child.get("entPhysicalModelName") or "").strip() + if model: + items.append(child) + # Always recurse to find deeper items (containers may lack models) + self._collect_children(child["entPhysicalIndex"], inventory_data, items) + + def _get_module_types(self): + """Get all module types indexed by model, with mappings applied.""" + from dcim.models import ModuleType + + from netbox_librenms_plugin.models import ModuleTypeMapping + + types = ModuleType.objects.all().select_related("manufacturer") + result = {} + for mt in types: + result[mt.model] = mt + if mt.part_number and mt.part_number != mt.model: + result[mt.part_number] = mt + for mapping in ModuleTypeMapping.objects.select_related("netbox_module_type__manufacturer"): + result[mapping.librenms_model] = mapping.netbox_module_type + return result + + def _install_single(self, device, item, index_map, module_types, ModuleBay, ModuleType, Module): + """Try to install a single inventory item. + + Re-fetches module bays each time since parent installs create new ones. + Scopes bay lookup to the correct parent module to handle duplicate bay names. + """ + from netbox_librenms_plugin.models import ModuleBayMapping + from netbox_librenms_plugin.utils import apply_normalization_rules + + model_name = (item.get("entPhysicalModelName") or "").strip() + serial = (item.get("entPhysicalSerialNum") or "").strip() + name = item.get("entPhysicalName", "") or model_name + + # Match module type (direct, then normalization fallback) + matched_type = module_types.get(model_name) + if not matched_type and model_name: + manufacturer = getattr(getattr(device, "device_type", None), "manufacturer", None) + normalized = apply_normalization_rules(model_name, "module_type", manufacturer=manufacturer) + if normalized != model_name: + matched_type = module_types.get(normalized) + if not matched_type: + return {"status": "skipped", "name": name, "reason": "no matching type"} + + # Re-fetch module bays (parent install creates new child bays) + bays = ModuleBay.objects.filter(device=device).select_related("installed_module__module_type") + + # Determine if this item belongs under an installed module + # by tracing its LibreNMS parent hierarchy to an installed item + parent_module_id = self._find_parent_module_id(item, index_map, device, ModuleBay) + + if parent_module_id: + bay_dict = {bay.name: bay for bay in bays if bay.module_id == parent_module_id} + else: + bay_dict = {bay.name: bay for bay in bays if not bay.module_id} + + # Match module bay using mapping table + matched_bay = self._match_bay(item, index_map, bay_dict, ModuleBayMapping) + if not matched_bay: + return {"status": "skipped", "name": name, "reason": "no matching bay"} + + # Check if already installed + if hasattr(matched_bay, "installed_module") and matched_bay.installed_module: + return {"status": "skipped", "name": name, "reason": "bay already occupied"} + + # Install + try: + with transaction.atomic(): # savepoint: failure here won't abort parent tx + module = Module( + device=device, + module_bay=matched_bay, + module_type=matched_type, + serial=serial, + status="active", + ) + module.full_clean() + module.save() + except Exception as e: + error_msg = str(e) + if "dcim_interface_unique_device_name" in error_msg: + error_msg = ( + "duplicate interface name — this module type's interface template " + "uses {module} which resolves to the same name for all siblings. " + "An interface naming plugin with a rewrite rule for this module type can fix this." + ) + return {"status": "failed", "name": name, "reason": error_msg} + + return {"status": "installed", "name": f"{matched_type.model} → {matched_bay.name}"} + + @staticmethod + def _find_parent_module_id(item, index_map, device, ModuleBay): + """Find the NetBox module ID for the installed parent of this inventory item. + + Walks up the LibreNMS hierarchy to find an ancestor whose name matches + an installed module bay on the device. + """ + from netbox_librenms_plugin.models import ModuleBayMapping + + current = item + for _ in range(10): # max depth guard + parent_idx = current.get("entPhysicalContainedIn", 0) + if not parent_idx or parent_idx not in index_map: + return None + parent = index_map[parent_idx] + parent_name = parent.get("entPhysicalName", "") + parent_descr = parent.get("entPhysicalDescr", "") + + # Check if this parent matches an installed module bay on the device + device_bays = ModuleBay.objects.filter(device=device, module_id__isnull=True).select_related( + "installed_module" + ) + + for bay in device_bays: + if hasattr(bay, "installed_module") and bay.installed_module: + if bay.name == parent_name or (parent_descr and bay.name == parent_descr): + return bay.installed_module.pk + + # Also check ModuleBayMapping for indirect matches + for name in [parent_name, parent_descr]: + if not name: + continue + mapping = ModuleBayMapping.objects.filter(librenms_name=name).first() + if mapping: + bay = ( + ModuleBay.objects.filter(device=device, name=mapping.netbox_bay_name, module_id__isnull=True) + .select_related("installed_module") + .first() + ) + if bay and hasattr(bay, "installed_module") and bay.installed_module: + return bay.installed_module.pk + + current = parent + return None + + @staticmethod + def _match_bay(item, index_map, module_bays, ModuleBayMapping): + """Match an inventory item to a module bay (same logic as BaseModuleTableView).""" + import re + + # Resolve parent name + contained_in = item.get("entPhysicalContainedIn", 0) + parent_name = None + if contained_in: + parent = index_map.get(contained_in) + if parent: + parent_name = parent.get("entPhysicalName", "") + + item_name = item.get("entPhysicalName", "") + item_descr = item.get("entPhysicalDescr", "") + phys_class = item.get("entPhysicalClass", "") + + # Build candidate names: parent, item name, item description + candidate_names = [n for n in [parent_name, item_name, item_descr] if n] + + # Check mapping for each candidate (exact match) + for name in candidate_names: + filters = {"librenms_name": name, "is_regex": False} + if phys_class: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class=phys_class).first() + if not mapping: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + else: + mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + if mapping and mapping.netbox_bay_name in module_bays: + return module_bays[mapping.netbox_bay_name] + + # Regex pattern matching on all candidate names + for name in candidate_names: + bay = BaseModuleTableView._lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping) + if bay: + return bay + + # Fallback: exact match on candidate names against bay dict + for name in candidate_names: + if name in module_bays: + return module_bays[name] + + # Positional fallback for items inside converters + return BaseModuleTableView._match_bay_by_position(item, index_map, module_bays) diff --git a/netbox_librenms_plugin/views/imports/list.py b/netbox_librenms_plugin/views/imports/list.py index ca8ba15871..e077b6de58 100644 --- a/netbox_librenms_plugin/views/imports/list.py +++ b/netbox_librenms_plugin/views/imports/list.py @@ -279,6 +279,30 @@ def get(self, request, *args, **kwargs): # noqa: D401 - inherited doc logger.error(f"Error getting device count: {e}") device_count = 0 + # Load settings for background job decision; resolve naming preferences. + # We intentionally read user_pref here rather than request.GET because the + # naming toggles (use-sysname-toggle, strip-domain-toggle) live OUTSIDE the + # filter form (method="get") and are not submitted with it. Instead, each + # toggle fires a savePref() AJAX call on change, so the user_pref is always + # up-to-date by the time the filter form is submitted. + settings = None + try: + settings = LibreNMSSettings.objects.first() + except Exception: + logger.exception("Failed to load LibreNMSSettings for background job naming prefs") + _use_sysname_pref = get_user_pref(request, "plugins.netbox_librenms_plugin.use_sysname") + _strip_domain_pref = get_user_pref(request, "plugins.netbox_librenms_plugin.strip_domain") + _use_sysname = ( + _use_sysname_pref + if _use_sysname_pref is not None + else (getattr(settings, "use_sysname_default", True) if settings else True) + ) + _strip_domain = ( + _strip_domain_pref + if _strip_domain_pref is not None + else (getattr(settings, "strip_domain_default", False) if settings else False) + ) + # Decide whether to use background job # Skip background job if data is already cached if not devices_cached and self.should_use_background_job(): @@ -295,8 +319,8 @@ def get(self, request, *args, **kwargs): # noqa: D401 - inherited doc show_disabled=bool(self._filter_form_data.get("show_disabled")), exclude_existing=bool(self._filter_form_data.get("exclude_existing")), server_key=self.librenms_api.server_key, - use_sysname=self._use_sysname, - strip_domain=self._strip_domain, + use_sysname=_use_sysname, + strip_domain=_strip_domain, ) logger.info( @@ -328,6 +352,26 @@ def get(self, request, *args, **kwargs): # noqa: D401 - inherited doc filter_warning = self._filter_warning + # Load settings for import defaults + try: + settings, _ = LibreNMSSettings.objects.get_or_create() + except Exception: + _user = getattr(request, "user", None) + logger.exception( + "Failed to get or create LibreNMSSettings during LibreNMS import for user %s", + getattr(_user, "username", str(_user)), + ) + settings = None + + # User preference overrides for toggles (persisted per-user) + use_sysname = get_user_pref(request, "plugins.netbox_librenms_plugin.use_sysname") + strip_domain = get_user_pref(request, "plugins.netbox_librenms_plugin.strip_domain") + # Fall back to server-level settings + if use_sysname is None: + use_sysname = getattr(settings, "use_sysname_default", True) if settings else True + if strip_domain is None: + strip_domain = getattr(settings, "strip_domain_default", False) if settings else False + # Get active cached searches for this server cached_searches = get_active_cached_searches(self.librenms_api.server_key) @@ -420,6 +464,32 @@ def _get_import_queryset(self): show_disabled = bool(data_source.get("show_disabled")) exclude_existing = bool(data_source.get("exclude_existing")) + # Resolve naming preferences: submitted form toggle → user pref → settings default. + # When pref is None (first-time user) any explicit toggle in data_source should still win. + use_sysname_pref = get_user_pref(self._request, "plugins.netbox_librenms_plugin.use_sysname") + strip_domain_pref = get_user_pref(self._request, "plugins.netbox_librenms_plugin.strip_domain") + try: + _settings = LibreNMSSettings.objects.first() + except Exception: + logger.exception("Failed to load LibreNMSSettings for naming preferences") + _settings = None + _use_sysname_toggle = data_source.get("use_sysname_toggle") + use_sysname = ( + _use_sysname_toggle + if _use_sysname_toggle is not None + else use_sysname_pref + if use_sysname_pref is not None + else (getattr(_settings, "use_sysname_default", True) if _settings else True) + ) + _strip_domain_toggle = data_source.get("strip_domain_toggle") + strip_domain = ( + _strip_domain_toggle + if _strip_domain_toggle is not None + else strip_domain_pref + if strip_domain_pref is not None + else (getattr(_settings, "strip_domain_default", False) if _settings else False) + ) + validated_devices, from_cache = process_device_filters( api=self.librenms_api, filters=libre_filters, @@ -429,8 +499,8 @@ def _get_import_queryset(self): exclude_existing=exclude_existing, request=self._request, return_cache_status=True, - use_sysname=self._use_sysname, - strip_domain=self._strip_domain, + use_sysname=use_sysname, + strip_domain=strip_domain, ) self._from_cache = from_cache @@ -444,8 +514,8 @@ def _get_import_queryset(self): server_key=self.librenms_api.server_key, filters=libre_filters, vc_enabled=vc_detection_enabled, - use_sysname=self._use_sysname, - strip_domain=self._strip_domain, + use_sysname=use_sysname, + strip_domain=strip_domain, ) cache_metadata = cache.get(cache_metadata_key) if cache_metadata: diff --git a/netbox_librenms_plugin/views/mapping_views.py b/netbox_librenms_plugin/views/mapping_views.py index b1fcec9c77..55ff7bd658 100644 --- a/netbox_librenms_plugin/views/mapping_views.py +++ b/netbox_librenms_plugin/views/mapping_views.py @@ -1,14 +1,44 @@ from netbox.views import generic from utilities.views import register_model_view -from netbox_librenms_plugin.filters import InterfaceTypeMappingFilterSet +from netbox_librenms_plugin.filters import ( + DeviceTypeMappingFilterSet, + InterfaceTypeMappingFilterSet, + ModuleBayMappingFilterSet, + ModuleTypeMappingFilterSet, + NormalizationRuleFilterSet, +) from netbox_librenms_plugin.forms import ( + DeviceTypeMappingFilterForm, + DeviceTypeMappingForm, + DeviceTypeMappingImportForm, InterfaceTypeMappingFilterForm, InterfaceTypeMappingForm, InterfaceTypeMappingImportForm, + ModuleBayMappingFilterForm, + ModuleBayMappingForm, + ModuleBayMappingImportForm, + ModuleTypeMappingFilterForm, + ModuleTypeMappingForm, + ModuleTypeMappingImportForm, + NormalizationRuleFilterForm, + NormalizationRuleForm, + NormalizationRuleImportForm, +) +from netbox_librenms_plugin.models import ( + DeviceTypeMapping, + InterfaceTypeMapping, + ModuleBayMapping, + ModuleTypeMapping, + NormalizationRule, +) +from netbox_librenms_plugin.tables.mappings import ( + DeviceTypeMappingTable, + InterfaceTypeMappingTable, + ModuleBayMappingTable, + ModuleTypeMappingTable, + NormalizationRuleTable, ) -from netbox_librenms_plugin.models import InterfaceTypeMapping -from netbox_librenms_plugin.tables.mappings import InterfaceTypeMappingTable from netbox_librenms_plugin.views.mixins import LibreNMSPermissionMixin @@ -84,3 +114,243 @@ class InterfaceTypeMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectC """ queryset = InterfaceTypeMapping.objects.all() + + +# --- DeviceTypeMapping views --- + + +class DeviceTypeMappingListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all DeviceTypeMapping objects.""" + + queryset = DeviceTypeMapping.objects.all() + table = DeviceTypeMappingTable + filterset = DeviceTypeMappingFilterSet + filterset_form = DeviceTypeMappingFilterForm + template_name = "netbox_librenms_plugin/devicetypemapping_list.html" + + +class DeviceTypeMappingCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + form = DeviceTypeMappingForm + + +@register_model_view(DeviceTypeMapping, "bulk_import", path="import", detail=False) +class DeviceTypeMappingBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing DeviceTypeMapping objects.""" + + queryset = DeviceTypeMapping.objects.all() + model_form = DeviceTypeMappingImportForm + + +class DeviceTypeMappingView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + + +class DeviceTypeMappingEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + form = DeviceTypeMappingForm + + +class DeviceTypeMappingDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + + +class DeviceTypeMappingBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple DeviceTypeMapping objects.""" + + queryset = DeviceTypeMapping.objects.all() + table = DeviceTypeMappingTable + + +class DeviceTypeMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific DeviceTypeMapping object.""" + + queryset = DeviceTypeMapping.objects.all() + + +# --- ModuleTypeMapping views --- + + +class ModuleTypeMappingListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all ModuleTypeMapping objects.""" + + queryset = ModuleTypeMapping.objects.all() + table = ModuleTypeMappingTable + filterset = ModuleTypeMappingFilterSet + filterset_form = ModuleTypeMappingFilterForm + template_name = "netbox_librenms_plugin/moduletypemapping_list.html" + + +class ModuleTypeMappingCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + form = ModuleTypeMappingForm + + +@register_model_view(ModuleTypeMapping, "bulk_import", path="import", detail=False) +class ModuleTypeMappingBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing ModuleTypeMapping objects.""" + + queryset = ModuleTypeMapping.objects.all() + model_form = ModuleTypeMappingImportForm + + +class ModuleTypeMappingView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + + +class ModuleTypeMappingEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + form = ModuleTypeMappingForm + + +class ModuleTypeMappingDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + + +class ModuleTypeMappingBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple ModuleTypeMapping objects.""" + + queryset = ModuleTypeMapping.objects.all() + table = ModuleTypeMappingTable + + +class ModuleTypeMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific ModuleTypeMapping object.""" + + queryset = ModuleTypeMapping.objects.all() + + +# --- ModuleBayMapping views --- + + +class ModuleBayMappingListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all ModuleBayMapping objects.""" + + queryset = ModuleBayMapping.objects.all() + table = ModuleBayMappingTable + filterset = ModuleBayMappingFilterSet + filterset_form = ModuleBayMappingFilterForm + template_name = "netbox_librenms_plugin/modulebaymapping_list.html" + + +class ModuleBayMappingCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + form = ModuleBayMappingForm + + +@register_model_view(ModuleBayMapping, "bulk_import", path="import", detail=False) +class ModuleBayMappingBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing ModuleBayMapping objects.""" + + queryset = ModuleBayMapping.objects.all() + model_form = ModuleBayMappingImportForm + + +class ModuleBayMappingView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + + +class ModuleBayMappingEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + form = ModuleBayMappingForm + + +class ModuleBayMappingDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + + +class ModuleBayMappingBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple ModuleBayMapping objects.""" + + queryset = ModuleBayMapping.objects.all() + table = ModuleBayMappingTable + + +class ModuleBayMappingChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific ModuleBayMapping object.""" + + queryset = ModuleBayMapping.objects.all() + + +# --- NormalizationRule views --- + + +class NormalizationRuleListView(LibreNMSPermissionMixin, generic.ObjectListView): + """Provides a view for listing all NormalizationRule objects.""" + + queryset = NormalizationRule.objects.all() + table = NormalizationRuleTable + filterset = NormalizationRuleFilterSet + filterset_form = NormalizationRuleFilterForm + template_name = "netbox_librenms_plugin/normalizationrule_list.html" + + +class NormalizationRuleCreateView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for creating a new NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + form = NormalizationRuleForm + + +@register_model_view(NormalizationRule, "bulk_import", path="import", detail=False) +class NormalizationRuleBulkImportView(LibreNMSPermissionMixin, generic.BulkImportView): + """Provides a view for bulk importing NormalizationRule objects.""" + + queryset = NormalizationRule.objects.all() + model_form = NormalizationRuleImportForm + + +class NormalizationRuleView(LibreNMSPermissionMixin, generic.ObjectView): + """Provides a view for displaying details of a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + + +class NormalizationRuleEditView(LibreNMSPermissionMixin, generic.ObjectEditView): + """Provides a view for editing a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + form = NormalizationRuleForm + + +class NormalizationRuleDeleteView(LibreNMSPermissionMixin, generic.ObjectDeleteView): + """Provides a view for deleting a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() + + +class NormalizationRuleBulkDeleteView(LibreNMSPermissionMixin, generic.BulkDeleteView): + """Provides a view for deleting multiple NormalizationRule objects.""" + + queryset = NormalizationRule.objects.all() + table = NormalizationRuleTable + + +class NormalizationRuleChangeLogView(LibreNMSPermissionMixin, generic.ObjectChangeLogView): + """Provides a view for displaying the change log of a specific NormalizationRule object.""" + + queryset = NormalizationRule.objects.all() diff --git a/netbox_librenms_plugin/views/object_sync/__init__.py b/netbox_librenms_plugin/views/object_sync/__init__.py index e9893cf7d8..f025cb2a7b 100644 --- a/netbox_librenms_plugin/views/object_sync/__init__.py +++ b/netbox_librenms_plugin/views/object_sync/__init__.py @@ -5,6 +5,7 @@ DeviceInterfaceTableView, DeviceIPAddressTableView, DeviceLibreNMSSyncView, + DeviceModuleTableView, DeviceVLANTableView, SaveVlanGroupOverridesView, SingleInterfaceVerifyView, diff --git a/netbox_librenms_plugin/views/object_sync/devices.py b/netbox_librenms_plugin/views/object_sync/devices.py index 429a3119a8..fe12c42751 100644 --- a/netbox_librenms_plugin/views/object_sync/devices.py +++ b/netbox_librenms_plugin/views/object_sync/devices.py @@ -17,6 +17,7 @@ LibreNMSInterfaceTable, VCInterfaceTable, ) +from netbox_librenms_plugin.tables.modules import LibreNMSModuleTable from netbox_librenms_plugin.utils import ( get_interface_name_field, get_missing_vlan_warning, @@ -29,6 +30,7 @@ from ..base.interfaces_view import BaseInterfaceTableView from ..base.ip_addresses_view import BaseIPAddressTableView from ..base.librenms_sync_view import BaseLibreNMSSyncView +from ..base.modules_view import BaseModuleTableView from ..base.vlan_table_view import BaseVLANTableView from ..mixins import CacheMixin, LibreNMSPermissionMixin @@ -63,6 +65,12 @@ def get_vlan_context(self, request, obj): vlan_table_view.request = request return vlan_table_view.get_vlan_context(request, obj) + def get_module_context(self, request, obj): + """Return module sync context for the device.""" + module_table_view = DeviceModuleTableView() + module_table_view.request = request + return module_table_view.get_context_data(request, obj) + class DeviceInterfaceTableView(BaseInterfaceTableView): """Interface synchronization table for Devices.""" @@ -384,3 +392,15 @@ class DeviceVLANTableView(BaseVLANTableView): """VLAN synchronization table view for Devices.""" model = Device + + +class DeviceModuleTableView(BaseModuleTableView): + """Module/inventory synchronization view for Devices.""" + + model = Device + + def get_table(self, data, obj): + """Return the module sync table.""" + table = LibreNMSModuleTable(data, device=obj) + table.htmx_url = f"{self.request.path}?tab=modules" + return table diff --git a/netbox_librenms_plugin/views/sync/cables.py b/netbox_librenms_plugin/views/sync/cables.py index 0e52f3b017..27dc5943c2 100644 --- a/netbox_librenms_plugin/views/sync/cables.py +++ b/netbox_librenms_plugin/views/sync/cables.py @@ -1,3 +1,5 @@ +import logging + from dcim.models import Cable, Device, Interface from django.contrib import messages from django.core.cache import cache @@ -9,6 +11,8 @@ from netbox_librenms_plugin.views.mixins import CacheMixin, LibreNMSPermissionMixin, NetBoxObjectPermissionMixin +logger = logging.getLogger(__name__) + class SyncCablesView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, CacheMixin, View): """Create NetBox cables using cached LibreNMS link data.""" @@ -42,7 +46,11 @@ def get_cached_links_data(self, request, obj): return cached_data.get("links", []) def create_cable(self, local_interface, remote_interface, request): - """Create a cable between local and remote interfaces.""" + """Create a cable between local and remote interfaces. + + Returns: + True on success, False on failure. + """ try: Cable.objects.create( a_terminations=[local_interface], @@ -81,13 +89,12 @@ def process_single_interface(self, interface, cached_links): link_data = next(link for link in cached_links if link["local_port"] == interface["interface"]) return self.handle_cable_creation(link_data, interface) except StopIteration: - return {"status": "invalid"} + return {"status": "invalid", "interface": interface.get("interface", "")} def verify_cable_creation_requirements(self, link_data): """Return True if all required NetBox IDs are present in link data.""" required_fields = [ "netbox_local_interface_id", - "netbox_remote_device_id", "netbox_remote_interface_id", ] @@ -113,13 +120,21 @@ def handle_cable_creation(self, link_data, interface): return {"status": "missing_remote", "interface": interface["interface"]} def process_interface_sync(self, selected_interfaces, cached_links): - """Process cable sync for all selected interfaces and return results.""" + """Process cable sync for all selected interfaces and return results. + + Each interface is processed in its own atomic block so individual + failures roll back only that cable without affecting others. + """ results = {"valid": [], "invalid": [], "duplicate": [], "missing_remote": []} - with transaction.atomic(): - for interface in selected_interfaces: - result = self.process_single_interface(interface, cached_links) + for interface in selected_interfaces: + try: + with transaction.atomic(): + result = self.process_single_interface(interface, cached_links) results[result["status"]].append(result.get("interface", "")) + except Exception: + logger.exception("Failed to sync cable for interface %s", interface.get("interface", "")) + results["invalid"].append(interface.get("interface", "")) return results diff --git a/netbox_librenms_plugin/views/sync/devices.py b/netbox_librenms_plugin/views/sync/devices.py index da0f9af5b0..eb45b375ff 100644 --- a/netbox_librenms_plugin/views/sync/devices.py +++ b/netbox_librenms_plugin/views/sync/devices.py @@ -26,7 +26,7 @@ def get_object(self, object_id): try: return Device.objects.get(pk=object_id) except Device.DoesNotExist: - return VirtualMachine.objects.get(pk=object_id) + return get_object_or_404(VirtualMachine, pk=object_id) def post(self, request, object_id): """Add a device to LibreNMS using the submitted SNMP form.""" diff --git a/netbox_librenms_plugin/views/sync/interfaces.py b/netbox_librenms_plugin/views/sync/interfaces.py index f5d92ab0a2..89bbc5ac92 100644 --- a/netbox_librenms_plugin/views/sync/interfaces.py +++ b/netbox_librenms_plugin/views/sync/interfaces.py @@ -164,14 +164,8 @@ def sync_interface(self, obj, librenms_interface, exclude_columns, interface_nam ) # Sync VLANs if not excluded - vlan_synced = False if "vlans" not in exclude_columns: self._sync_interface_vlans(interface, librenms_interface, interface_name) - vlan_synced = True - - # Skip redundant save when _sync_interface_vlans already saved (via _update_interface_vlan_assignment) - if not vlan_synced: - interface.save() def get_netbox_interface_type(self, librenms_interface): """Return the NetBox interface type mapped from LibreNMS type and speed.""" diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 0000000000..62c68691ec --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,6 @@ +"""Conftest for e2e tests — no Django initialization needed.""" + +import os + +# Prevent pytest-django from trying to initialize Django +os.environ.pop("DJANGO_SETTINGS_MODULE", None) diff --git a/tests/e2e/test_module_install.py b/tests/e2e/test_module_install.py new file mode 100644 index 0000000000..cee05eb904 --- /dev/null +++ b/tests/e2e/test_module_install.py @@ -0,0 +1,330 @@ +"""End-to-end Playwright tests for LibreNMS plugin module sync workflow. + +These tests exercise the full import → modules → install flow against a +live NetBox + LibreNMS instance inside the devcontainer. + +Prerequisites: + - NetBox running at NETBOX_URL (default http://172.22.0.4:8000) + - LibreNMS server configured in plugin settings + - Device 15 (WS-C4900M) exists and is linked to LibreNMS + - Playwright installed: pip install playwright && playwright install chromium + +Run: + cd /home/mzieba/workspace/netbox-librenms-plugin + HTTP_PROXY= HTTPS_PROXY= http_proxy= https_proxy= \ + no_proxy=localhost,127.0.0.1,172.22.0.4 \ + python -m pytest tests/e2e/test_module_install.py -v -s +""" + +import os +import subprocess +import time + +import pytest + +NETBOX_URL = os.environ.get("NETBOX_URL", "http://172.22.0.4:8000") +NETBOX_USER = os.environ.get("NETBOX_USER", "admin") +NETBOX_PASS = os.environ.get("NETBOX_PASS", "admin") +CONTAINER_NAME = None + + +def _get_container(): + """Find the devcontainer name.""" + global CONTAINER_NAME + if CONTAINER_NAME: + return CONTAINER_NAME + result = subprocess.run( + ["docker", "ps", "--format", "{{.Names}}"], + capture_output=True, + text=True, + ) + for name in result.stdout.strip().split("\n"): + if "devcontainer-devcontainer" in name: + CONTAINER_NAME = name + return name + pytest.skip("No devcontainer found") + + +def _netbox_shell(code): + """Run Python code in NetBox's Django shell.""" + import shlex + + container = _get_container() + escaped = shlex.quote(code) + result = subprocess.run( + [ + "docker", + "exec", + container, + "bash", + "-c", + f"cd /opt/netbox/netbox && python3 manage.py shell -c {escaped}", + ], + capture_output=True, + text=True, + env={"PATH": "/usr/bin:/bin", "HOME": "/root"}, + ) + # Filter out config loading lines + lines = [line for line in result.stdout.strip().split("\n") if not line.startswith(("🧬", "156 objects"))] + return "\n".join(lines).strip() + + +def _delete_device_modules(device_id): + """Remove all modules from a device.""" + _netbox_shell( + f"from dcim.models import Module; " + f"deleted = Module.objects.filter(device_id={device_id}).delete(); " + f"print(f'Deleted {{deleted}}')" + ) + + +def _get_interfaces(device_id): + """Get interface names for a device.""" + output = _netbox_shell( + f"from dcim.models import Interface; " + f'[print(f\'{{i.name}}|{{i.module.module_type.model if i.module else "-"}}|' + f'{{i.module.module_bay.name if i.module else "-"}}\')' + f" for i in Interface.objects.filter(device_id={device_id}).order_by('name')]" + ) + results = [] + for line in output.split("\n"): + if "|" in line: + name, mod_type, bay = line.split("|") + results.append({"name": name, "module_type": mod_type, "bay": bay}) + return results + + +@pytest.fixture(scope="module") +def browser(): + """Launch browser for the test module.""" + from playwright.sync_api import sync_playwright + + pw = sync_playwright().start() + b = pw.chromium.launch(headless=True) + yield b + b.close() + pw.stop() + + +@pytest.fixture +def page(browser): + """Create a new page and log in to NetBox.""" + ctx = browser.new_context(ignore_https_errors=True) + pg = ctx.new_page() + + pg.goto(f"{NETBOX_URL}/login/", timeout=10000) + pg.fill("#id_username", NETBOX_USER) + pg.fill("#id_password", NETBOX_PASS) + pg.click("button[type=submit]") + pg.wait_for_load_state("networkidle") + yield pg + ctx.close() + + +class TestModuleInstallWorkflow: + """Test the full module sync and install workflow on device 15 (WS-C4900M).""" + + DEVICE_ID = 15 + + def _goto_modules_tab(self, page): + """Navigate to the modules sync tab and refresh data.""" + page.goto(f"{NETBOX_URL}/dcim/devices/{self.DEVICE_ID}/librenms-sync/?tab=modules") + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Click Refresh Modules + btn = page.query_selector('button:has-text("Refresh Modules")') + assert btn is not None, "Refresh Modules button not found" + btn.click() + time.sleep(8) + + def _get_table_rows(self, page): + """Parse the module sync table into dicts.""" + pane = page.query_selector("#modules") + assert pane is not None, "Modules pane not found" + + rows = [] + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if len(cells) >= 8: + rows.append( + { + "name": cells[0].inner_text().strip(), + "model": cells[1].inner_text().strip(), + "serial": cells[2].inner_text().strip(), + "bay": cells[5].inner_text().strip(), + "type": cells[6].inner_text().strip(), + "status": cells[7].inner_text().strip(), + } + ) + return rows + + def test_clean_state_shows_install_buttons(self, page): + """After deleting all modules, table shows Install buttons.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + rows = self._get_table_rows(page) + assert len(rows) > 0, "No rows in module sync table" + + # Top-level items with matched bays should show Matched status + supervisor = [r for r in rows if "Supervisor(slot 1)" in r["name"]] + assert len(supervisor) == 1, f"Expected 1 Supervisor row, got {len(supervisor)}" + assert supervisor[0]["status"] == "Matched", f"Expected Matched, got {supervisor[0]['status']}" + + def test_single_install(self, page): + """Installing a single top-level module works.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + # Install FanTray 1 + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "FanTray 1" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install"):not(:has-text("Branch"))') + if btn: + btn.click() + time.sleep(5) + break + + # Verify via DB + output = _netbox_shell( + f"from dcim.models import Module; " + f"m = Module.objects.filter(device_id={self.DEVICE_ID}, module_bay__name='Fan Tray 1').first(); " + f"print(m.module_type.model if m else 'NONE')" + ) + assert "WS-X4992" in output, f"FanTray not installed: {output}" + + def test_branch_install_supervisor(self, page): + """Branch install creates supervisor + X2 transceivers with correct names.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + # Click Install Branch on Supervisor(slot 1) + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Supervisor(slot 1)" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install Branch")') + assert btn is not None, "Install Branch button not found for Supervisor" + btn.click() + break + + # Wait for branch install to complete (creates many modules + signals) + time.sleep(20) + page.wait_for_load_state("networkidle") + time.sleep(5) + + # Verify interfaces have correct names (not bare position numbers) + interfaces = _get_interfaces(self.DEVICE_ID) + x2_interfaces = [i for i in interfaces if i["module_type"] in ("X2-10GB-LR", "X2-10GB-SR")] + + assert len(x2_interfaces) > 0, "No X2 transceiver interfaces created" + + for iface in x2_interfaces: + assert iface["name"].startswith("TenGigabitEthernet"), ( + f"Interface '{iface['name']}' in {iface['bay']} " + f"should start with 'TenGigabitEthernet' (INR rule not applied?)" + ) + + def test_branch_install_no_duplicate_errors(self, page): + """Branch install handles already-occupied bays gracefully.""" + # Don't delete modules — some should already be installed + self._goto_modules_tab(page) + + # Click Install Branch on Supervisor(slot 1) again + pane = page.query_selector("#modules") + branch_btn = None + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Supervisor(slot 1)" in cells[0].inner_text(): + branch_btn = tr.query_selector('button:has-text("Install Branch")') + break + + if branch_btn: + branch_btn.click() + time.sleep(10) + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Check for error messages — should only have skips, no failures + body_text = page.query_selector("body").inner_text() + assert "Branch install failed" not in body_text, ( + "Branch install crashed instead of handling errors gracefully" + ) + + def test_child_bays_hidden_when_parent_not_installed(self, page): + """Children show 'No Bay' when parent module is not installed.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + rows = self._get_table_rows(page) + + # Children of Supervisor(slot 1) should show "No matching bay" + # since Supervisor isn't installed, its child bays don't exist yet + children = [r for r in rows if r["name"].startswith("└─") and "TenGigabitEthernet1/" in r["name"]] + for child in children: + assert "No matching bay" in child["bay"], ( + f"Child '{child['name']}' should show 'No matching bay' when parent not installed, got '{child['bay']}'" + ) + + def test_full_workflow(self, page): + """Full workflow: clean → install individuals → branch install → verify.""" + _delete_device_modules(self.DEVICE_ID) + self._goto_modules_tab(page) + + # Step 1: Install PSUs and FanTray individually + for label in ["FanTray 1", "Power Supply 1", "Power Supply 2"]: + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and label in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install"):not(:has-text("Branch"))') + if btn: + btn.click() + time.sleep(4) + break + + # Step 2: Branch install Supervisor + transceivers + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Supervisor(slot 1)" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install Branch")') + if btn: + btn.click() + time.sleep(10) + break + + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Step 3: Branch install Linecard + self._goto_modules_tab(page) + pane = page.query_selector("#modules") + for tr in pane.query_selector_all("table tr"): + cells = tr.query_selector_all("td") + if cells and "Linecard(slot 3)" in cells[0].inner_text(): + btn = tr.query_selector('button:has-text("Install Branch")') + if btn: + btn.click() + time.sleep(10) + break + + page.wait_for_load_state("networkidle") + time.sleep(2) + + # Verify: all installable modules should be installed + self._goto_modules_tab(page) + rows = self._get_table_rows(page) + + matched_but_not_installed = [r for r in rows if r["status"] == "Matched" and not r["name"].startswith("└─")] + assert len(matched_but_not_installed) == 0, ( + f"Top-level items still 'Matched' after full workflow: {[r['name'] for r in matched_but_not_installed]}" + ) + + # Verify interface naming + interfaces = _get_interfaces(self.DEVICE_ID) + for iface in interfaces: + assert iface["name"] != "1", "Interface with bare name '1' found — INR rule not applied" From 298a99658c586587e9629b9d2a90afef7a1e360c Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Wed, 4 Mar 2026 18:08:17 +0100 Subject: [PATCH 06/28] fix: address PR review findings (sync from librenms_id-rebased) --- .../import_utils/__init__.py | 1 + .../import_utils/bulk_import.py | 15 ++++++++--- netbox_librenms_plugin/import_utils/cache.py | 26 +++++++++++++++++++ .../import_utils/filters.py | 22 +++++++++------- .../js/librenms_import.js | 10 ++----- netbox_librenms_plugin/utils.py | 10 ++++++- .../views/base/ip_addresses_view.py | 10 +++---- .../views/base/librenms_sync_view.py | 9 +++++++ .../views/imports/actions.py | 21 ++++++++++++--- .../views/sync/device_fields.py | 8 +++--- 10 files changed, 100 insertions(+), 32 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/__init__.py b/netbox_librenms_plugin/import_utils/__init__.py index b7a08fef59..81c24025ea 100644 --- a/netbox_librenms_plugin/import_utils/__init__.py +++ b/netbox_librenms_plugin/import_utils/__init__.py @@ -23,6 +23,7 @@ get_active_cached_searches, get_cache_metadata_key, get_import_device_cache_key, + get_import_search_cache_key, get_validated_device_cache_key, ) from .device_operations import ( # noqa: F401 diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 21105cb3bb..79b16fab2d 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -21,6 +21,15 @@ logger = logging.getLogger(__name__) +def _safe_disabled(device: dict) -> int: + """Return 1 if the device is disabled, 0 otherwise. Tolerates None/non-numeric values.""" + val = device.get("disabled", 0) + try: + return int(val) + except (TypeError, ValueError): + return 0 + + def bulk_import_devices_shared( device_ids: List[int], server_key: str = None, @@ -312,7 +321,7 @@ def _refresh_existing_device(validation: dict, libre_device: dict = None, server if refreshed: validation["existing_device"] = refreshed if hasattr(refreshed, "role") and refreshed.role: - validation["device_role"] = {"found": True, "role": refreshed.role} + validation.setdefault("device_role", {}).update({"found": True, "role": refreshed.role}) else: # Device was deleted since caching — recompute readiness to match # validate_device_for_import logic. @@ -370,7 +379,7 @@ def _refresh_existing_device(validation: dict, libre_device: dict = None, server if not new_device and sys_name: new_device = Model.objects.filter(name__iexact=sys_name).first() if new_device: - match_type = "hostname" + match_type = "sysname" if new_device: validation["existing_device"] = new_device @@ -450,7 +459,7 @@ def process_device_filters( # 0=enabled) reflects manual device disablement; "status" reflects SNMP reachability. # show_disabled controls the former: hidden when disabled==1, shown regardless of status. if not show_disabled: - libre_devices = [d for d in libre_devices if int(d.get("disabled", 0)) != 1] + libre_devices = [d for d in libre_devices if _safe_disabled(d) != 1] if job: job.logger.info(f"Found {len(libre_devices)} devices to process") diff --git a/netbox_librenms_plugin/import_utils/cache.py b/netbox_librenms_plugin/import_utils/cache.py index 140f45f79d..119bb111b2 100644 --- a/netbox_librenms_plugin/import_utils/cache.py +++ b/netbox_librenms_plugin/import_utils/cache.py @@ -184,3 +184,29 @@ def get_import_device_cache_key(device_id: int | str, server_key: str = "default 'import_device_data_production_123' """ return f"import_device_data_{server_key}_{device_id}" + + +def get_import_search_cache_key(server_key: str, api_filters: dict, client_filters: dict) -> str: + """ + Generate a deterministic cache key for a LibreNMS device search result. + + The key encodes the server, API-side filters, and client-side filters so + that different filter combinations produce distinct cache entries. + + Args: + server_key: Resolved LibreNMS server key (use ``api.server_key``). + api_filters: Filters forwarded to the LibreNMS API. + client_filters: Filters applied client-side after the API response. + + Returns: + str: Cache key for the import search result. + """ + import hashlib + import json + + def _hash(d): + return hashlib.sha256( + json.dumps(sorted(d.items()) if isinstance(d, dict) else d, sort_keys=True).encode() + ).hexdigest()[:16] + + return f"librenms_devices_import_{server_key}_{_hash(api_filters)}_{_hash(client_filters)}" diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py index 3e12658066..1ec418e8e4 100644 --- a/netbox_librenms_plugin/import_utils/filters.py +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -1,17 +1,26 @@ """Device filtering and retrieval from LibreNMS.""" -import hashlib -import json import logging from typing import List from django.core.cache import cache +from .cache import get_import_search_cache_key + from ..librenms_api import LibreNMSAPI logger = logging.getLogger(__name__) +def _safe_disabled(device: dict) -> int: + """Return 1 if the device is disabled, 0 otherwise. Tolerates None/non-numeric values.""" + val = device.get("disabled", 0) + try: + return int(val) + except (TypeError, ValueError): + return 0 + + def get_device_count_for_filters( api: LibreNMSAPI, filters: dict, @@ -39,7 +48,7 @@ def get_device_count_for_filters( # 0=enabled) reflects manual device disablement; "status" reflects SNMP reachability. # show_disabled controls the former: hidden when disabled==1, shown regardless of status. if not show_disabled: - devices = [d for d in devices if int(d.get("disabled", 0)) != 1] + devices = [d for d in devices if _safe_disabled(d) != 1] return len(devices) @@ -181,12 +190,7 @@ def get_librenms_devices_for_import( # Use caching to avoid repeated API calls # Include both API and client filters in cache key (deterministic, cross-process stable). # Use api.server_key (always resolved) rather than the raw server_key arg (may differ). - def _hash(d): - return hashlib.sha256( - json.dumps(sorted(d.items()) if isinstance(d, dict) else d, sort_keys=True).encode() - ).hexdigest()[:16] - - cache_key = f"librenms_devices_import_{api.server_key}_{_hash(api_filters)}_{_hash(client_filters)}" + cache_key = get_import_search_cache_key(api.server_key, api_filters, client_filters) from_cache = False if force_refresh: diff --git a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js index 2aeef777af..1e3a05f649 100644 --- a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js +++ b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js @@ -1095,17 +1095,11 @@ // Initialize Bootstrap tooltips inside the freshly-swapped modal content if (typeof bootstrap !== 'undefined' && bootstrap.Tooltip) { - const tooltips = modalContent.querySelectorAll('[data-bs-toggle="tooltip"]'); - [...tooltips].forEach(el => new bootstrap.Tooltip(el)); + const tooltipEls = modalContent.querySelectorAll('[data-bs-toggle="tooltip"]'); + tooltipEls.forEach(el => bootstrap.Tooltip.getOrCreateInstance(el)); } showModal(modalElement, fallbackBackdropRef); - - // Re-initialize tooltips for newly swapped modal content - if (typeof bootstrap !== 'undefined' && bootstrap.Tooltip) { - const tooltipEls = modalContent.querySelectorAll('[data-bs-toggle="tooltip"]'); - [...tooltipEls].map(el => new bootstrap.Tooltip(el)); - } } document.body.addEventListener('htmx:afterSwap', ensureModalVisible); diff --git a/netbox_librenms_plugin/utils.py b/netbox_librenms_plugin/utils.py index c92dd66727..a5eb282708 100644 --- a/netbox_librenms_plugin/utils.py +++ b/netbox_librenms_plugin/utils.py @@ -517,7 +517,15 @@ def set_librenms_device_id(obj, device_id, server_key: str = "default"): obj, ) cf_value = {} - cf_value[server_key] = device_id + try: + cf_value[server_key] = int(device_id) + except (TypeError, ValueError): + logger.warning( + "librenms_id device_id %r is not a valid integer on %r; storing as-is.", + device_id, + obj, + ) + cf_value[server_key] = device_id obj.custom_field_data["librenms_id"] = cf_value diff --git a/netbox_librenms_plugin/views/base/ip_addresses_view.py b/netbox_librenms_plugin/views/base/ip_addresses_view.py index 1ca3f6e290..26c2dafc27 100644 --- a/netbox_librenms_plugin/views/base/ip_addresses_view.py +++ b/netbox_librenms_plugin/views/base/ip_addresses_view.py @@ -105,11 +105,11 @@ def _prefetch_netbox_data(self, obj): # Create maps for efficient lookups server_key = self.librenms_api.server_key - interfaces_by_librenms_id = { - get_librenms_device_id(interface, server_key): interface - for interface in all_interfaces - if get_librenms_device_id(interface, server_key) - } + interfaces_by_librenms_id = {} + for interface in all_interfaces: + lib_id = get_librenms_device_id(interface, server_key) + if lib_id: + interfaces_by_librenms_id[lib_id] = interface interfaces_by_name = {interface.name: interface for interface in all_interfaces} diff --git a/netbox_librenms_plugin/views/base/librenms_sync_view.py b/netbox_librenms_plugin/views/base/librenms_sync_view.py index c723859904..0e405deb4e 100644 --- a/netbox_librenms_plugin/views/base/librenms_sync_view.py +++ b/netbox_librenms_plugin/views/base/librenms_sync_view.py @@ -151,6 +151,15 @@ def _build_all_server_mappings(obj, active_server_key): result = [] for sk, did in cf_value.items(): srv_cfg = servers_config.get(sk) + # Legacy single-server config: "default" key with no matching servers entry — + # fall back to root-level librenms_url/display_name in plugins_cfg. + if srv_cfg is None and sk == "default": + legacy_url = plugins_cfg.get("librenms_url") + if legacy_url: + srv_cfg = { + "librenms_url": legacy_url, + "display_name": plugins_cfg.get("display_name") or sk, + } is_configured = srv_cfg is not None librenms_url = srv_cfg.get("librenms_url") if srv_cfg else None display_name = (srv_cfg.get("display_name") or sk) if srv_cfg else sk diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 75ae108be2..9b0f345af8 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -1222,9 +1222,24 @@ def post(self, request, device_id): "Serial number not confirmed. Check the force checkbox to migrate without serial verification.", status=400, ) - migrate_legacy_librenms_id(existing_device, self.librenms_api.server_key) - if err := _save_device(existing_device): - return err + with transaction.atomic(): + try: + locked_device = Device.objects.select_for_update().get(pk=existing_device.pk) + except Device.DoesNotExist: + return HttpResponse( + "Device no longer exists; it may have been deleted concurrently.", + status=409, + ) + # Re-check under lock — another request may have already migrated it + cf_locked = locked_device.custom_field_data.get("librenms_id") + if not isinstance(cf_locked, int): + return HttpResponse( + "Device librenms_id is already in JSON format; no migration needed.", + status=400, + ) + migrate_legacy_librenms_id(locked_device, self.librenms_api.server_key) + if err := _save_device(locked_device): + return err logger.info( f"Migrated legacy librenms_id on '{existing_device.name}' " f"to {{{self.librenms_api.server_key!r}: {cf_value}}}" diff --git a/netbox_librenms_plugin/views/sync/device_fields.py b/netbox_librenms_plugin/views/sync/device_fields.py index d621c7cec6..180b3ccf13 100644 --- a/netbox_librenms_plugin/views/sync/device_fields.py +++ b/netbox_librenms_plugin/views/sync/device_fields.py @@ -459,11 +459,13 @@ def post(self, request, pk): device_locked.save() except ValidationError as exc: transaction.set_rollback(True) - messages.error(request, f"Validation error removing mapping: {exc}") + logger.error("Validation error removing LibreNMS mapping for server %r: %s", server_key, exc) + messages.error(request, "Validation error removing LibreNMS mapping.") return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) - except Exception as exc: + except Exception: transaction.set_rollback(True) - messages.error(request, f"Error removing mapping for server '{server_key}': {exc}") + logger.exception("Unexpected error removing LibreNMS mapping for server %r", server_key) + messages.error(request, "An unexpected error occurred while removing the LibreNMS mapping.") return redirect("plugins:netbox_librenms_plugin:device_librenms_sync", pk=pk) messages.success(request, f"Removed LibreNMS mapping for server '{server_key}'.") else: From 6c1938ef99a10751d81a2c42670f8a64681c251a Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Wed, 4 Mar 2026 19:09:19 +0100 Subject: [PATCH 07/28] fix: second batch of PR review findings (sync from librenms_id-rebased) --- .../import_utils/bulk_import.py | 14 ++++++++- .../import_utils/device_operations.py | 12 +++++-- .../import_utils/filters.py | 14 ++++++++- .../import_utils/virtual_chassis.py | 16 ++++++---- .../js/librenms_import.js | 2 +- .../htmx/device_validation_details.html | 2 +- .../librenms_sync_base.html | 2 +- netbox_librenms_plugin/utils.py | 4 +-- .../views/base/cables_view.py | 31 +++++++------------ .../views/base/librenms_sync_view.py | 2 +- .../views/imports/actions.py | 13 +++++++- 11 files changed, 74 insertions(+), 38 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index 79b16fab2d..dcfbfa5cba 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -22,8 +22,20 @@ def _safe_disabled(device: dict) -> int: - """Return 1 if the device is disabled, 0 otherwise. Tolerates None/non-numeric values.""" + """Return 1 if the device is disabled, 0 otherwise. + + Handles None, booleans, numeric strings, and common truthy/falsy tokens + (e.g. "true"/"yes"/"on" → 1, "false"/"no"/"off" → 0) without raising. + """ val = device.get("disabled", 0) + if isinstance(val, bool): + return int(val) + if isinstance(val, str): + normalized = val.strip().lower() + if normalized in ("1", "true", "yes", "on"): + return 1 + if normalized in ("0", "false", "no", "off", ""): + return 0 try: return int(val) except (TypeError, ValueError): diff --git a/netbox_librenms_plugin/import_utils/device_operations.py b/netbox_librenms_plugin/import_utils/device_operations.py index 487c915cf1..36bef26f33 100644 --- a/netbox_librenms_plugin/import_utils/device_operations.py +++ b/netbox_librenms_plugin/import_utils/device_operations.py @@ -254,12 +254,18 @@ def validate_device_for_import( device_id=librenms_id, ) result["resolved_name"] = hostname + _raw_sysname = libre_device.get("sysName") or "" + _raw_hostname = libre_device.get("hostname") or "" + if use_sysname: + _source = "sysname" if _raw_sysname else "hostname" + else: + _source = "hostname" if _raw_hostname else ("sysname" if _raw_sysname else "hostname") result["naming_criteria"] = { "use_sysname": use_sysname, "strip_domain": strip_domain, - "raw_sysname": libre_device.get("sysName") or "", - "raw_hostname": libre_device.get("hostname") or "", - "source": "sysname" if use_sysname and libre_device.get("sysName") else "hostname", + "raw_sysname": _raw_sysname, + "raw_hostname": _raw_hostname, + "source": _source, } logger.debug( f"Checking for existing device/VM: " diff --git a/netbox_librenms_plugin/import_utils/filters.py b/netbox_librenms_plugin/import_utils/filters.py index 1ec418e8e4..7b5c554589 100644 --- a/netbox_librenms_plugin/import_utils/filters.py +++ b/netbox_librenms_plugin/import_utils/filters.py @@ -13,8 +13,20 @@ def _safe_disabled(device: dict) -> int: - """Return 1 if the device is disabled, 0 otherwise. Tolerates None/non-numeric values.""" + """Return 1 if the device is disabled, 0 otherwise. + + Handles None, booleans, numeric strings, and common truthy/falsy tokens + (e.g. "true"/"yes"/"on" → 1, "false"/"no"/"off" → 0) without raising. + """ val = device.get("disabled", 0) + if isinstance(val, bool): + return int(val) + if isinstance(val, str): + normalized = val.strip().lower() + if normalized in ("1", "true", "yes", "on"): + return 1 + if normalized in ("0", "false", "no", "off", ""): + return 0 try: return int(val) except (TypeError, ValueError): diff --git a/netbox_librenms_plugin/import_utils/virtual_chassis.py b/netbox_librenms_plugin/import_utils/virtual_chassis.py index b95f32eaec..537598695a 100644 --- a/netbox_librenms_plugin/import_utils/virtual_chassis.py +++ b/netbox_librenms_plugin/import_utils/virtual_chassis.py @@ -34,7 +34,8 @@ def _clone_virtual_chassis_data(data: dict | None) -> dict: member_copy = member.copy() raw_position = member_copy.get("position", idx + 1) try: - member_copy["position"] = int(raw_position) + pos = int(raw_position) + member_copy["position"] = pos if pos > 0 else idx + 1 except (TypeError, ValueError): member_copy["position"] = idx + 1 # 1-based fallback; position 0 is invalid members.append(member_copy) @@ -307,13 +308,16 @@ def update_vc_member_suggested_names(vc_data: dict, master_name: str) -> dict: # Load naming pattern once to avoid a DB query per member vc_pattern = _load_vc_member_name_pattern() for idx, member in enumerate(vc_data.get("members", [])): - raw_position = member.get("position", idx) + # Positions are stored as 1-based (from entPhysicalParentRelPos or idx+1 fallback). + # Use them directly for name generation; only replace 0/negative with 1-based fallback. + raw_position = member.get("position", idx + 1) try: - base_position = int(raw_position) + position = int(raw_position) + if position <= 0: + position = idx + 1 except (TypeError, ValueError): - base_position = idx - position = base_position + 1 # Convert to 1-based position - member["position"] = base_position + position = idx + 1 + member["position"] = position member["suggested_name"] = _generate_vc_member_name( master_name, position, serial=member.get("serial"), pattern=vc_pattern ) diff --git a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js index 1e3a05f649..68fb8d2641 100644 --- a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js +++ b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_import.js @@ -1096,7 +1096,7 @@ // Initialize Bootstrap tooltips inside the freshly-swapped modal content if (typeof bootstrap !== 'undefined' && bootstrap.Tooltip) { const tooltipEls = modalContent.querySelectorAll('[data-bs-toggle="tooltip"]'); - tooltipEls.forEach(el => bootstrap.Tooltip.getOrCreateInstance(el)); + for (const el of tooltipEls) { bootstrap.Tooltip.getOrCreateInstance(el); } } showModal(modalElement, fallbackBackdropRef); diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html index 9ff9fc4c81..47f4b101fb 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/htmx/device_validation_details.html @@ -190,7 +190,7 @@
{% endif %} {% elif validation.device_type.device_type %} - + {{ validation.device_type.device_type }} diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html index 0350bf9dff..a4d6535152 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html @@ -75,7 +75,7 @@
+ onsubmit="return confirm('Remove mapping for server \'{{ mapping.server_key|escapejs }}\'? This cannot be undone.');"> {% csrf_token %}
+ +
+ {% include 'netbox_librenms_plugin/inc/paginator.html' with table=module_sync.table %} + {% include 'inc/table.html' with table=module_sync.table %} + {% include 'netbox_librenms_plugin/inc/paginator.html' with table=module_sync.table %} +
+ {% else %}
diff --git a/netbox_librenms_plugin/tests/test_integration_sync.py b/netbox_librenms_plugin/tests/test_integration_sync.py index 6bfb58f0e0..563d105d44 100644 --- a/netbox_librenms_plugin/tests/test_integration_sync.py +++ b/netbox_librenms_plugin/tests/test_integration_sync.py @@ -149,3 +149,79 @@ def test_add_device_failure_on_server_error(self, mock_server): ) assert success is False + + +class TestLibreNMSAPIInventory: + """LibreNMSAPI.get_device_inventory() correctly parses mock server responses.""" + + def test_returns_inventory_list(self, mock_server): + inventory = [ + { + "entPhysicalIndex": 1, + "entPhysicalDescr": "Chassis", + "entPhysicalClass": "chassis", + "entPhysicalSerialNum": "SN-CHASSIS-001", + "entPhysicalModelName": "WS-C4900M", + "entPhysicalName": "Chassis 1", + "entPhysicalContainedIn": 0, + }, + { + "entPhysicalIndex": 2, + "entPhysicalDescr": "Linecard", + "entPhysicalClass": "module", + "entPhysicalSerialNum": "SN-CARD-002", + "entPhysicalModelName": "WS-X4748-RJ45V+E", + "entPhysicalName": "Slot 1", + "entPhysicalContainedIn": 1, + }, + ] + mock_server.register("/api/v0/inventory/7/all", {"status": "ok", "inventory": inventory}) + api = _make_api(mock_server.url) + + success, data = api.get_device_inventory(7) + + assert success is True + assert isinstance(data, list) + assert len(data) == 2 + assert data[0]["entPhysicalClass"] == "chassis" + assert data[1]["entPhysicalModelName"] == "WS-X4748-RJ45V+E" + + def test_returns_empty_list_when_no_inventory(self, mock_server): + mock_server.register("/api/v0/inventory/99/all", {"status": "ok", "inventory": []}) + api = _make_api(mock_server.url) + + success, data = api.get_device_inventory(99) + + assert success is True + assert data == [] + + def test_returns_false_on_network_error(self, mock_server): + # Unregistered path → 404 → raise_for_status → RequestException + api = _make_api(mock_server.url) + + success, _ = api.get_device_inventory(404) + + assert success is False + + def test_inventory_items_preserve_all_fields(self, mock_server): + inventory = [ + { + "entPhysicalIndex": 5, + "entPhysicalDescr": "10 Gigabit Ethernet Module", + "entPhysicalClass": "module", + "entPhysicalSerialNum": "JAE123XYZ", + "entPhysicalModelName": "X2-10GB-LR", + "entPhysicalName": "TenGigabitEthernet1/1", + "entPhysicalContainedIn": 1, + "entPhysicalParentRelPos": 1, + } + ] + mock_server.register("/api/v0/inventory/3/all", {"status": "ok", "inventory": inventory}) + api = _make_api(mock_server.url) + + success, data = api.get_device_inventory(3) + + assert success is True + item = data[0] + assert item["entPhysicalParentRelPos"] == 1 + assert item["entPhysicalSerialNum"] == "JAE123XYZ" diff --git a/netbox_librenms_plugin/tests/test_sync_modules.py b/netbox_librenms_plugin/tests/test_sync_modules.py index 273c8f32d2..7fbb4c9ec6 100644 --- a/netbox_librenms_plugin/tests/test_sync_modules.py +++ b/netbox_librenms_plugin/tests/test_sync_modules.py @@ -1,6 +1,8 @@ -"""Tests for InstallModuleView and InstallBranchView (views/sync/modules.py). +"""Tests for module sync views and BaseModuleTableView bay matching logic. -inventory-rebased branch only. +Covers: InstallModuleView/InstallBranchView wiring, branch collection, cycle guards, +bay matching by name/mapping/position, serial comparison, status determination, +and depth tracking. inventory-rebased branch only. """ from unittest.mock import MagicMock, patch @@ -190,3 +192,566 @@ def test_has_netbox_object_permission_mixin(self): from netbox_librenms_plugin.views.mixins import NetBoxObjectPermissionMixin assert NetBoxObjectPermissionMixin in InstallBranchView.__mro__ + + +# --------------------------------------------------------------------------- +# Helper: build a BaseModuleTableView instance without __init__ +# --------------------------------------------------------------------------- + + +def _make_base_view(): + from netbox_librenms_plugin.views.base.modules_view import BaseModuleTableView + + view = object.__new__(BaseModuleTableView) + view._device_manufacturer = None + return view + + +def _bay(name, installed_module=None, pk=None): + """Quick MagicMock module bay.""" + bay = MagicMock() + bay.name = name + bay.pk = pk or id(bay) + bay.installed_module = installed_module + bay.get_absolute_url.return_value = f"/dcim/module-bays/{bay.pk}/" + return bay + + +def _module(serial="SN001"): + mod = MagicMock() + mod.serial = serial + mod.get_absolute_url.return_value = "/dcim/modules/1/" + return mod + + +# --------------------------------------------------------------------------- +# _determine_status +# --------------------------------------------------------------------------- + + +class TestDetermineStatus: + """_determine_status returns the correct badge string for every combination.""" + + def test_matched_bay_and_type(self): + view = _make_base_view() + assert view._determine_status(MagicMock(), MagicMock(), "") == "Matched" + + def test_no_bay_regardless_of_type(self): + view = _make_base_view() + assert view._determine_status(None, MagicMock(), "") == "No Bay" + assert view._determine_status(None, None, "") == "No Bay" + + def test_bay_without_type(self): + view = _make_base_view() + assert view._determine_status(MagicMock(), None, "") == "No Type" + + def test_unmatched_when_neither(self): + # This path is unreachable via current code (No Bay catches it first), + # but _determine_status is a standalone method so test the logic directly. + view = _make_base_view() + # Trick: pass a falsy non-None bay to skip "no bay" but reach "no type" + # Not possible with current logic; just verify No Bay path dominates. + assert view._determine_status(None, None, "SN1") == "No Bay" + + +# --------------------------------------------------------------------------- +# Serial comparison inside _build_row +# --------------------------------------------------------------------------- + + +class TestBuildRowSerialComparison: + """_build_row sets 'Installed' or 'Serial Mismatch' based on installed module serial.""" + + def _make_item(self, model_name, serial): + return { + "entPhysicalModelName": model_name, + "entPhysicalSerialNum": serial, + "entPhysicalName": model_name, + "entPhysicalDescr": "", + "entPhysicalClass": "module", + "entPhysicalIndex": 10, + "entPhysicalContainedIn": 0, + } + + def _make_matched_type(self, model="WS-X4748"): + mt = MagicMock() + mt.model = model + mt.pk = 1 + mt.get_absolute_url.return_value = "/dcim/module-types/1/" + # Make uses-module-path/token checks return False so badges don't appear + mt.interfacetemplates = MagicMock() + mt.interfacetemplates.all.return_value = [] + return mt + + def test_matching_serial_gives_installed_status(self): + view = _make_base_view() + item = self._make_item("WS-X4748", "SN-ABC-123") + mt = self._make_matched_type() + installed = _module(serial="SN-ABC-123") + bay = _bay("Slot 1", installed_module=installed) + + module_bays = {"Slot 1": bay} + module_types = {"WS-X4748": mt} + index_map = {10: item} + + with patch.object(view, "_match_module_bay", return_value=bay): + with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="WS-X4748"): + with patch("netbox_librenms_plugin.utils.module_type_uses_module_path", return_value=False): + with patch("netbox_librenms_plugin.utils.supports_module_path", return_value=False): + with patch("netbox_librenms_plugin.utils.has_nested_name_conflict", return_value=False): + with patch("netbox_librenms_plugin.utils.module_type_is_end_module", return_value=False): + with patch( + "netbox_librenms_plugin.utils.module_type_uses_module_token", return_value=False + ): + row = view._build_row(item, index_map, module_bays, module_types, depth=0) + + assert row["status"] == "Installed" + assert row["row_class"] == "table-success" + + def test_serial_mismatch_gives_danger_status(self): + view = _make_base_view() + item = self._make_item("WS-X4748", "SN-NEW-999") + mt = self._make_matched_type() + installed = _module(serial="SN-OLD-111") + bay = _bay("Slot 1", installed_module=installed) + + module_bays = {"Slot 1": bay} + module_types = {"WS-X4748": mt} + index_map = {10: item} + + with patch.object(view, "_match_module_bay", return_value=bay): + with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="WS-X4748"): + with patch("netbox_librenms_plugin.utils.module_type_uses_module_path", return_value=False): + with patch("netbox_librenms_plugin.utils.supports_module_path", return_value=False): + with patch("netbox_librenms_plugin.utils.has_nested_name_conflict", return_value=False): + with patch("netbox_librenms_plugin.utils.module_type_is_end_module", return_value=False): + with patch( + "netbox_librenms_plugin.utils.module_type_uses_module_token", return_value=False + ): + row = view._build_row(item, index_map, module_bays, module_types, depth=0) + + assert row["status"] == "Serial Mismatch" + assert row["row_class"] == "table-danger" + + def test_no_bay_gives_no_bay_status(self): + view = _make_base_view() + item = self._make_item("WS-X4748", "SN1") + mt = self._make_matched_type() + + with patch.object(view, "_match_module_bay", return_value=None): + with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="WS-X4748"): + with patch("netbox_librenms_plugin.utils.module_type_uses_module_path", return_value=False): + with patch("netbox_librenms_plugin.utils.supports_module_path", return_value=False): + with patch("netbox_librenms_plugin.utils.has_nested_name_conflict", return_value=False): + with patch("netbox_librenms_plugin.utils.module_type_is_end_module", return_value=False): + with patch( + "netbox_librenms_plugin.utils.module_type_uses_module_token", return_value=False + ): + row = view._build_row(item, {10: item}, {}, {"WS-X4748": mt}, depth=0) + + assert row["status"] == "No Bay" + + def test_no_type_gives_no_type_status(self): + view = _make_base_view() + item = self._make_item("UNKNOWN-MODEL", "SN1") + bay = _bay("Slot 1") + + with patch.object(view, "_match_module_bay", return_value=bay): + with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="UNKNOWN-MODEL"): + with patch("netbox_librenms_plugin.utils.module_type_uses_module_path", return_value=False): + with patch("netbox_librenms_plugin.utils.supports_module_path", return_value=False): + with patch("netbox_librenms_plugin.utils.has_nested_name_conflict", return_value=False): + with patch("netbox_librenms_plugin.utils.module_type_is_end_module", return_value=False): + with patch( + "netbox_librenms_plugin.utils.module_type_uses_module_token", return_value=False + ): + row = view._build_row(item, {10: item}, {"Slot 1": bay}, {}, depth=0) + + assert row["status"] == "No Type" + + def test_can_install_set_when_bay_free_and_type_matched(self): + """can_install=True only when bay exists, type matched, and bay is empty.""" + view = _make_base_view() + item = self._make_item("WS-X4748", "SN1") + mt = self._make_matched_type() + # Bay with no installed module + bay = _bay("Slot 1", installed_module=None) + bay.installed_module = None + + with patch.object(view, "_match_module_bay", return_value=bay): + with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="WS-X4748"): + with patch("netbox_librenms_plugin.utils.module_type_uses_module_path", return_value=False): + with patch("netbox_librenms_plugin.utils.supports_module_path", return_value=False): + with patch("netbox_librenms_plugin.utils.has_nested_name_conflict", return_value=False): + with patch("netbox_librenms_plugin.utils.module_type_is_end_module", return_value=False): + with patch( + "netbox_librenms_plugin.utils.module_type_uses_module_token", return_value=False + ): + row = view._build_row(item, {10: item}, {"Slot 1": bay}, {"WS-X4748": mt}, depth=0) + + assert row["can_install"] is True + + def test_can_install_false_when_bay_occupied(self): + view = _make_base_view() + item = self._make_item("WS-X4748", "SN1") + mt = self._make_matched_type() + installed = _module(serial="SN1") + bay = _bay("Slot 1", installed_module=installed) + + with patch.object(view, "_match_module_bay", return_value=bay): + with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="WS-X4748"): + with patch("netbox_librenms_plugin.utils.module_type_uses_module_path", return_value=False): + with patch("netbox_librenms_plugin.utils.supports_module_path", return_value=False): + with patch("netbox_librenms_plugin.utils.has_nested_name_conflict", return_value=False): + with patch("netbox_librenms_plugin.utils.module_type_is_end_module", return_value=False): + with patch( + "netbox_librenms_plugin.utils.module_type_uses_module_token", return_value=False + ): + row = view._build_row(item, {10: item}, {"Slot 1": bay}, {"WS-X4748": mt}, depth=0) + + assert row["can_install"] is False + + +# --------------------------------------------------------------------------- +# Depth tracking in render_name +# --------------------------------------------------------------------------- + + +class TestRenderNameDepth: + """render_name applies tree indentation based on depth.""" + + def test_depth_zero_returns_plain_value(self): + from netbox_librenms_plugin.tables.modules import LibreNMSModuleTable + + table = LibreNMSModuleTable([]) + result = table.render_name("Supervisor", {"depth": 0}) + assert "padding-left" not in str(result) + assert "Supervisor" in str(result) + + def test_depth_one_adds_padding(self): + from netbox_librenms_plugin.tables.modules import LibreNMSModuleTable + + table = LibreNMSModuleTable([]) + result = str(table.render_name("Line Card", {"depth": 1})) + assert "padding-left" in result + assert "20px" in result + + def test_depth_two_doubles_padding(self): + from netbox_librenms_plugin.tables.modules import LibreNMSModuleTable + + table = LibreNMSModuleTable([]) + result = str(table.render_name("SFP", {"depth": 2})) + assert "40px" in result + + def test_depth_renders_tree_prefix(self): + from netbox_librenms_plugin.tables.modules import LibreNMSModuleTable + + table = LibreNMSModuleTable([]) + result = str(table.render_name("Port 1", {"depth": 1})) + assert "└─" in result + + +# --------------------------------------------------------------------------- +# _match_bay_by_position +# --------------------------------------------------------------------------- + + +class TestMatchBayByPosition: + """_match_bay_by_position resolves position-based bay names for SFPs in converters.""" + + def test_matches_sfp_slot_by_sibling_order(self): + from netbox_librenms_plugin.views.base.modules_view import BaseModuleTableView + + # Build an inventory: parent (model) → container1 → item1, container2 → item2 + parent_item = { + "entPhysicalIndex": 1, + "entPhysicalModelName": "CONVERTER", + "entPhysicalContainedIn": 0, + "entPhysicalParentRelPos": 0, + } + container1 = { + "entPhysicalIndex": 2, + "entPhysicalModelName": "", + "entPhysicalContainedIn": 1, + "entPhysicalParentRelPos": 1, + } + container2 = { + "entPhysicalIndex": 3, + "entPhysicalModelName": "", + "entPhysicalContainedIn": 1, + "entPhysicalParentRelPos": 2, + } + sfp1 = { + "entPhysicalIndex": 4, + "entPhysicalModelName": "SFP-10G-LR", + "entPhysicalContainedIn": 2, + "entPhysicalParentRelPos": 1, + } + sfp2 = { + "entPhysicalIndex": 5, + "entPhysicalModelName": "SFP-10G-SR", + "entPhysicalContainedIn": 3, + "entPhysicalParentRelPos": 1, + } + + index_map = {1: parent_item, 2: container1, 3: container2, 4: sfp1, 5: sfp2} + bays = {"SFP 1": _bay("SFP 1"), "SFP 2": _bay("SFP 2")} + + result1 = BaseModuleTableView._match_bay_by_position(sfp1, index_map, bays) + result2 = BaseModuleTableView._match_bay_by_position(sfp2, index_map, bays) + + assert result1 is bays["SFP 1"] + assert result2 is bays["SFP 2"] + + def test_returns_none_when_no_modelless_container(self): + from netbox_librenms_plugin.views.base.modules_view import BaseModuleTableView + + # Item directly under parent with model (no modelless container) + parent = {"entPhysicalIndex": 1, "entPhysicalModelName": "PARENT", "entPhysicalContainedIn": 0} + item = {"entPhysicalIndex": 2, "entPhysicalModelName": "CHILD", "entPhysicalContainedIn": 1} + index_map = {1: parent, 2: item} + bays = {"Slot 1": _bay("Slot 1")} + + result = BaseModuleTableView._match_bay_by_position(item, index_map, bays) + assert result is None + + def test_returns_none_when_no_bays_match_pattern(self): + from netbox_librenms_plugin.views.base.modules_view import BaseModuleTableView + + parent = { + "entPhysicalIndex": 1, + "entPhysicalModelName": "M", + "entPhysicalContainedIn": 0, + "entPhysicalParentRelPos": 0, + } + container = { + "entPhysicalIndex": 2, + "entPhysicalModelName": "", + "entPhysicalContainedIn": 1, + "entPhysicalParentRelPos": 1, + } + item = { + "entPhysicalIndex": 3, + "entPhysicalModelName": "X", + "entPhysicalContainedIn": 2, + "entPhysicalParentRelPos": 1, + } + index_map = {1: parent, 2: container, 3: item} + bays = {"InterfaceA": _bay("InterfaceA")} # no "SFP 1"/"Slot 1"/etc. + + result = BaseModuleTableView._match_bay_by_position(item, index_map, bays) + assert result is None + + +# --------------------------------------------------------------------------- +# _match_module_bay — exact name fallback +# --------------------------------------------------------------------------- + + +class TestMatchModuleBayExactFallback: + """When no ModuleBayMapping exists, exact parent/item/descr name is tried.""" + + def test_exact_parent_name_match(self): + from netbox_librenms_plugin.views.base.modules_view import BaseModuleTableView + + view = _make_base_view() + parent = { + "entPhysicalIndex": 1, + "entPhysicalModelName": "PARENT", + "entPhysicalContainedIn": 0, + "entPhysicalName": "Slot 1", + } + item = { + "entPhysicalIndex": 2, + "entPhysicalName": "Linecard A", + "entPhysicalDescr": "", + "entPhysicalClass": "module", + "entPhysicalContainedIn": 1, + } + index_map = {1: parent, 2: item} + bay = _bay("Slot 1") + bays = {"Slot 1": bay} + + with patch("netbox_librenms_plugin.models.ModuleBayMapping") as mock_mbm: + mock_mbm.objects.filter.return_value.first.return_value = None + mock_mbm.objects.filter.return_value = MagicMock() + mock_mbm.objects.filter.return_value.first.return_value = None + + # Also patch _lookup_regex_bay_mapping to return None + with patch.object(BaseModuleTableView, "_lookup_regex_bay_mapping", return_value=None): + with patch.object(BaseModuleTableView, "_match_bay_by_position", return_value=None): + result = view._match_module_bay(item, index_map, bays) + + assert result is bay + + def test_item_name_used_when_no_parent_name(self): + from netbox_librenms_plugin.views.base.modules_view import BaseModuleTableView + + view = _make_base_view() + item = { + "entPhysicalIndex": 1, + "entPhysicalName": "Module Bay 3", + "entPhysicalDescr": "", + "entPhysicalClass": "module", + "entPhysicalContainedIn": 0, + } + index_map = {1: item} + bay = _bay("Module Bay 3") + bays = {"Module Bay 3": bay} + + with patch("netbox_librenms_plugin.models.ModuleBayMapping") as mock_mbm: + mock_mbm.objects.filter.return_value.first.return_value = None + with patch.object(BaseModuleTableView, "_lookup_regex_bay_mapping", return_value=None): + with patch.object(BaseModuleTableView, "_match_bay_by_position", return_value=None): + result = view._match_module_bay(item, index_map, bays) + + assert result is bay + + def test_returns_none_when_no_match(self): + from netbox_librenms_plugin.views.base.modules_view import BaseModuleTableView + + view = _make_base_view() + item = { + "entPhysicalIndex": 1, + "entPhysicalName": "Unknown-X", + "entPhysicalDescr": "", + "entPhysicalClass": "module", + "entPhysicalContainedIn": 0, + } + index_map = {1: item} + bays = {"Slot 1": _bay("Slot 1")} + + with patch("netbox_librenms_plugin.models.ModuleBayMapping") as mock_mbm: + mock_mbm.objects.filter.return_value.first.return_value = None + with patch.object(BaseModuleTableView, "_lookup_regex_bay_mapping", return_value=None): + with patch.object(BaseModuleTableView, "_match_bay_by_position", return_value=None): + result = view._match_module_bay(item, index_map, bays) + + assert result is None + + +# --------------------------------------------------------------------------- +# _install_single — status codes +# --------------------------------------------------------------------------- + + +class TestInstallSingleStatus: + """_install_single returns the correct status dict in each path.""" + + def _make_args(self): + """Return (device, item, index_map, module_types, ModuleBay, ModuleType, Module).""" + device = MagicMock() + device.device_type.manufacturer = None + + item = { + "entPhysicalIndex": 10, + "entPhysicalModelName": "WS-X4748", + "entPhysicalSerialNum": "SN123", + "entPhysicalName": "Line Card", + "entPhysicalContainedIn": 0, + } + + mt = MagicMock() + mt.model = "WS-X4748" + mt.pk = 1 + + bay = _bay("Slot 1") + bay.installed_module = None + + index_map = {10: item} + module_types = {"WS-X4748": mt} + + ModuleBay = MagicMock() + ModuleBay.objects.filter.return_value.select_related.return_value = [bay] + ModuleType = MagicMock() + Module = MagicMock() + + return device, item, index_map, module_types, ModuleBay, ModuleType, Module, bay, mt + + def test_returns_installed_on_success(self): + from contextlib import contextmanager + + view = _make_install_branch_view() + device, item, index_map, module_types, ModuleBay, ModuleType, Module, bay, mt = self._make_args() + module_instance = MagicMock() + Module.return_value = module_instance + + @contextmanager + def noop_atomic(): + yield + + with patch("netbox_librenms_plugin.views.sync.modules.transaction.atomic", noop_atomic): + with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="WS-X4748"): + with patch.object(view, "_find_parent_module_id", return_value=None): + with patch.object(view, "_match_bay", return_value=bay): + result = view._install_single( + device, item, index_map, module_types, ModuleBay, ModuleType, Module + ) + + assert result["status"] == "installed" + assert "WS-X4748" in result["name"] + + def test_returns_skipped_when_no_type(self): + view = _make_install_branch_view() + device, item, index_map, module_types, ModuleBay, ModuleType, Module, bay, mt = self._make_args() + + with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="WS-X4748"): + with patch.object(view, "_find_parent_module_id", return_value=None): + result = view._install_single( + device, + item, + index_map, + {}, # empty module_types → no match + ModuleBay, + ModuleType, + Module, + ) + + assert result["status"] == "skipped" + assert "no matching type" in result["reason"] + + def test_returns_skipped_when_no_bay(self): + view = _make_install_branch_view() + device, item, index_map, module_types, ModuleBay, ModuleType, Module, bay, mt = self._make_args() + + with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="WS-X4748"): + with patch.object(view, "_find_parent_module_id", return_value=None): + with patch.object(view, "_match_bay", return_value=None): + result = view._install_single(device, item, index_map, module_types, ModuleBay, ModuleType, Module) + + assert result["status"] == "skipped" + assert "no matching bay" in result["reason"] + + def test_returns_skipped_when_bay_already_occupied(self): + view = _make_install_branch_view() + device, item, index_map, module_types, ModuleBay, ModuleType, Module, bay, mt = self._make_args() + bay.installed_module = _module() # occupied! + + with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="WS-X4748"): + with patch.object(view, "_find_parent_module_id", return_value=None): + with patch.object(view, "_match_bay", return_value=bay): + result = view._install_single(device, item, index_map, module_types, ModuleBay, ModuleType, Module) + + assert result["status"] == "skipped" + assert "already occupied" in result["reason"] + + def test_returns_failed_on_exception(self): + from contextlib import contextmanager + + view = _make_install_branch_view() + device, item, index_map, module_types, ModuleBay, ModuleType, Module, bay, mt = self._make_args() + Module.side_effect = Exception("DB error") + + @contextmanager + def noop_atomic(): + yield + + with patch("netbox_librenms_plugin.views.sync.modules.transaction.atomic", noop_atomic): + with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="WS-X4748"): + with patch.object(view, "_find_parent_module_id", return_value=None): + with patch.object(view, "_match_bay", return_value=bay): + result = view._install_single( + device, item, index_map, module_types, ModuleBay, ModuleType, Module + ) + + assert result["status"] == "failed" diff --git a/netbox_librenms_plugin/urls.py b/netbox_librenms_plugin/urls.py index ace6126ebe..2d4a982602 100644 --- a/netbox_librenms_plugin/urls.py +++ b/netbox_librenms_plugin/urls.py @@ -31,6 +31,7 @@ DeviceVLANTableView, InstallBranchView, InstallModuleView, + InstallSelectedView, InterfaceTypeMappingBulkDeleteView, InterfaceTypeMappingBulkImportView, InterfaceTypeMappingChangeLogView, @@ -122,6 +123,11 @@ InstallBranchView.as_view(), name="install_branch", ), + path( + "devices//install-selected/", + InstallSelectedView.as_view(), + name="install_selected", + ), path( "devices//ipaddress-sync/", DeviceIPAddressTableView.as_view(), diff --git a/netbox_librenms_plugin/views/__init__.py b/netbox_librenms_plugin/views/__init__.py index 8bcceb5c17..cb62916c54 100644 --- a/netbox_librenms_plugin/views/__init__.py +++ b/netbox_librenms_plugin/views/__init__.py @@ -10,7 +10,7 @@ from .base.interfaces_view import BaseInterfaceTableView # noqa: F401 from .base.ip_addresses_view import BaseIPAddressTableView, SingleIPAddressVerifyView # noqa: F401 from .base.librenms_sync_view import BaseLibreNMSSyncView # noqa: F401 -from .sync.modules import InstallBranchView, InstallModuleView # noqa: F401 +from .sync.modules import InstallBranchView, InstallModuleView, InstallSelectedView # noqa: F401 from .base.vlan_table_view import BaseVLANTableView # noqa: F401 from .imports import ( # noqa: F401 BulkImportConfirmView, diff --git a/netbox_librenms_plugin/views/sync/modules.py b/netbox_librenms_plugin/views/sync/modules.py index d342ff59d9..f3256fd2dc 100644 --- a/netbox_librenms_plugin/views/sync/modules.py +++ b/netbox_librenms_plugin/views/sync/modules.py @@ -354,3 +354,82 @@ def _match_bay(item, index_map, module_bays, ModuleBayMapping): # Positional fallback for items inside converters return BaseModuleTableView._match_bay_by_position(item, index_map, module_bays) + + +class InstallSelectedView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, CacheMixin, View): + """Install a user-selected set of inventory items by their entPhysicalIndex values. + + Reuses InstallBranchView._install_single for each selected item so every item + goes through the same type/bay/serial resolution pipeline as a branch install. + Only items where a matching bay *and* module type are found will be installed; + items with no bay or no type are silently skipped (same behaviour as branch). + """ + + def post(self, request, pk): + from dcim.models import Device, Module, ModuleBay, ModuleType + + self.required_object_permissions = {"POST": [("add", Module)]} + if error := self.require_all_permissions_json("POST"): + return error + + device = get_object_or_404(Device, pk=pk) + + selected_indices = request.POST.getlist("select") + if not selected_indices: + messages.warning(request, "No modules selected.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + cached_data = cache.get(self.get_cache_key(device, "inventory")) + if not cached_data: + messages.error(request, "No cached inventory data. Please refresh modules first.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + try: + selected_set = {int(i) for i in selected_indices} + except ValueError: + messages.error(request, "Invalid selection.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + index_map = {item["entPhysicalIndex"]: item for item in cached_data} + items = [index_map[idx] for idx in selected_set if idx in index_map] + + if not items: + messages.warning(request, "None of the selected indices matched cached inventory.") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + helper = InstallBranchView() + module_types = helper._get_module_types() + + installed, skipped, failed = [], [], [] + + try: + with transaction.atomic(): + for item in items: + result = helper._install_single( + device, item, index_map, module_types, ModuleBay, ModuleType, Module + ) + if result["status"] == "installed": + installed.append(result["name"]) + elif result["status"] == "skipped": + skipped.append(f"{result['name']}: {result['reason']}") + else: + failed.append(f"{result['name']}: {result['reason']}") + except Exception as e: + messages.error(request, f"Install failed: {e}") + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") + + if installed: + cache.delete(self.get_cache_key(device, "inventory")) + messages.success(request, f"Installed {len(installed)} module(s): {', '.join(installed)}") + if skipped: + messages.info(request, f"Skipped {len(skipped)}: {'; '.join(skipped)}") + if failed: + messages.warning(request, f"Failed {len(failed)}: {'; '.join(failed)}") + + sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) + return redirect(f"{sync_url}?tab=modules#librenms-module-table") From 35f5a847387e40dd80e990e223b9f8abd2c51664 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 5 Mar 2026 14:55:34 +0100 Subject: [PATCH 19/28] fix: code review findings - template path, cache key, N+1, conflict check, tests - librenms_sync_base.html: use _module_sync.html top-level path (follow convention) - views/sync/modules.py: add CacheMixin to InstallModuleView, use get_cache_key - views/sync/modules.py: refactor _find_parent_module_id to accept pre-fetched device_bays/bay_mappings; update _install_single to pre-fetch and pass them, eliminating N+1 DB queries in the parent-module lookup loop - views/base/modules_view.py: seed visited={parent_idx} in _get_descendants to prevent self-referencing items being added as their own descendants - views/imports/actions.py: add Q OR to migrate_librenms_id conflict check so both namespaced and legacy integer librenms_id owners are detected - tests/test_mixins.py: change hasattr guard to explicit assert for vlan_overrides_key - tests/test_integration_sync.py: remove redundant url/token assignments in _make_api - tests/test_sync_modules.py: call real _get_module_types(); patch ModuleBayMapping in _install_single tests to avoid DB access --- .../netbox_librenms_plugin/_module_sync.html | 27 +++++++ .../librenms_sync_base.html | 2 +- .../tests/test_integration_sync.py | 3 - netbox_librenms_plugin/tests/test_mixins.py | 8 +- .../tests/test_sync_modules.py | 80 ++++++++++--------- .../views/base/modules_view.py | 2 +- .../views/imports/actions.py | 6 +- netbox_librenms_plugin/views/sync/modules.py | 41 ++++++---- 8 files changed, 108 insertions(+), 61 deletions(-) create mode 100644 netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync.html diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync.html new file mode 100644 index 0000000000..4c2c5ae65d --- /dev/null +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync.html @@ -0,0 +1,27 @@ +{% load helpers %} + + +
+

Module Sync

+
+
+ {% csrf_token %} + {% if has_librenms_id %} + {% with model_name=object|meta:"model_name" %} + {% if model_name == "device" %} + + {% endif %} + {% endwith %} + {% endif %} +
+
+
+ + +
+ {% include 'netbox_librenms_plugin/_module_sync_content.html' %} +
diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html index 0a3993aa79..91c49290fa 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/librenms_sync_base.html @@ -641,7 +641,7 @@
Device Information Sync
{% if module_sync %}
- {% include 'netbox_librenms_plugin/inc/_module_sync.html' %} + {% include 'netbox_librenms_plugin/_module_sync.html' %}
{% endif %} diff --git a/netbox_librenms_plugin/tests/test_integration_sync.py b/netbox_librenms_plugin/tests/test_integration_sync.py index 563d105d44..d281b513c7 100644 --- a/netbox_librenms_plugin/tests/test_integration_sync.py +++ b/netbox_librenms_plugin/tests/test_integration_sync.py @@ -35,9 +35,6 @@ def _make_api(url, token="test-token"): with patch("netbox_librenms_plugin.librenms_api.get_plugin_config") as mock_cfg: mock_cfg.side_effect = lambda _plugin, key: servers_config if key == "servers" else None api = LibreNMSAPI(server_key="test") - - api.librenms_url = url - api.api_token = token return api diff --git a/netbox_librenms_plugin/tests/test_mixins.py b/netbox_librenms_plugin/tests/test_mixins.py index 7d3b776246..9db2f7d2ba 100644 --- a/netbox_librenms_plugin/tests/test_mixins.py +++ b/netbox_librenms_plugin/tests/test_mixins.py @@ -167,7 +167,7 @@ def test_get_vlan_overrides_key_exists_and_differs_from_data_key(self): obj._meta.model_name = "device" obj.pk = 7 - if hasattr(mixin, "get_vlan_overrides_key"): - vlan_key = mixin.get_vlan_overrides_key(obj) - data_key = mixin.get_cache_key(obj, "vlans") - assert vlan_key != data_key + assert hasattr(mixin, "get_vlan_overrides_key"), "CacheMixin must implement get_vlan_overrides_key" + vlan_key = mixin.get_vlan_overrides_key(obj) + data_key = mixin.get_cache_key(obj, "vlans") + assert vlan_key != data_key diff --git a/netbox_librenms_plugin/tests/test_sync_modules.py b/netbox_librenms_plugin/tests/test_sync_modules.py index 7fbb4c9ec6..0d0f4c8062 100644 --- a/netbox_librenms_plugin/tests/test_sync_modules.py +++ b/netbox_librenms_plugin/tests/test_sync_modules.py @@ -123,27 +123,21 @@ def test_indexes_by_model_and_part_number(self): mock_mapping.librenms_model = "libre-model-a" mock_mapping.netbox_module_type = mt1 - with patch("dcim.models.ModuleType") as mock_mt_cls: - with patch("netbox_librenms_plugin.models.ModuleTypeMapping") as mock_map_cls: - mock_mt_cls.objects.all.return_value.select_related.return_value = [mt1, mt2] - mock_map_cls.objects.select_related.return_value = [mock_mapping] - - # _get_module_types imports inline, so patch at source - with patch.dict( - "sys.modules", - { - "dcim.models": type("m", (), {"ModuleType": mock_mt_cls})(), - }, - ): - pass # skip the complex mock — test the data structure instead - - # Test the indexing logic directly using a simplified version - result = {} - for mt in [mt1, mt2]: - result[mt.model] = mt - if mt.part_number and mt.part_number != mt.model: - result[mt.part_number] = mt - result[mock_mapping.librenms_model] = mock_mapping.netbox_module_type + mock_mt_cls = MagicMock() + mock_mt_cls.objects.all.return_value.select_related.return_value = [mt1, mt2] + + mock_map_cls = MagicMock() + mock_map_cls.objects.select_related.return_value = [mock_mapping] + + with patch.dict( + "sys.modules", + { + "dcim.models": type("m", (), {"ModuleType": mock_mt_cls})(), + }, + ): + with patch("netbox_librenms_plugin.models.ModuleTypeMapping", mock_map_cls): + view = _make_install_branch_view() + result = view._get_module_types() assert result["WS-X4748"] is mt1 assert result["ALT-PART-4748"] is mt1 @@ -682,11 +676,13 @@ def noop_atomic(): with patch("netbox_librenms_plugin.views.sync.modules.transaction.atomic", noop_atomic): with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="WS-X4748"): - with patch.object(view, "_find_parent_module_id", return_value=None): - with patch.object(view, "_match_bay", return_value=bay): - result = view._install_single( - device, item, index_map, module_types, ModuleBay, ModuleType, Module - ) + with patch("netbox_librenms_plugin.models.ModuleBayMapping") as mock_mapping_cls: + mock_mapping_cls.objects.all.return_value = [] + with patch.object(view, "_find_parent_module_id", return_value=None): + with patch.object(view, "_match_bay", return_value=bay): + result = view._install_single( + device, item, index_map, module_types, ModuleBay, ModuleType, Module + ) assert result["status"] == "installed" assert "WS-X4748" in result["name"] @@ -715,9 +711,13 @@ def test_returns_skipped_when_no_bay(self): device, item, index_map, module_types, ModuleBay, ModuleType, Module, bay, mt = self._make_args() with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="WS-X4748"): - with patch.object(view, "_find_parent_module_id", return_value=None): - with patch.object(view, "_match_bay", return_value=None): - result = view._install_single(device, item, index_map, module_types, ModuleBay, ModuleType, Module) + with patch("netbox_librenms_plugin.models.ModuleBayMapping") as mock_mapping_cls: + mock_mapping_cls.objects.all.return_value = [] + with patch.object(view, "_find_parent_module_id", return_value=None): + with patch.object(view, "_match_bay", return_value=None): + result = view._install_single( + device, item, index_map, module_types, ModuleBay, ModuleType, Module + ) assert result["status"] == "skipped" assert "no matching bay" in result["reason"] @@ -728,9 +728,13 @@ def test_returns_skipped_when_bay_already_occupied(self): bay.installed_module = _module() # occupied! with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="WS-X4748"): - with patch.object(view, "_find_parent_module_id", return_value=None): - with patch.object(view, "_match_bay", return_value=bay): - result = view._install_single(device, item, index_map, module_types, ModuleBay, ModuleType, Module) + with patch("netbox_librenms_plugin.models.ModuleBayMapping") as mock_mapping_cls: + mock_mapping_cls.objects.all.return_value = [] + with patch.object(view, "_find_parent_module_id", return_value=None): + with patch.object(view, "_match_bay", return_value=bay): + result = view._install_single( + device, item, index_map, module_types, ModuleBay, ModuleType, Module + ) assert result["status"] == "skipped" assert "already occupied" in result["reason"] @@ -748,10 +752,12 @@ def noop_atomic(): with patch("netbox_librenms_plugin.views.sync.modules.transaction.atomic", noop_atomic): with patch("netbox_librenms_plugin.utils.apply_normalization_rules", return_value="WS-X4748"): - with patch.object(view, "_find_parent_module_id", return_value=None): - with patch.object(view, "_match_bay", return_value=bay): - result = view._install_single( - device, item, index_map, module_types, ModuleBay, ModuleType, Module - ) + with patch("netbox_librenms_plugin.models.ModuleBayMapping") as mock_mapping_cls: + mock_mapping_cls.objects.all.return_value = [] + with patch.object(view, "_find_parent_module_id", return_value=None): + with patch.object(view, "_match_bay", return_value=bay): + result = view._install_single( + device, item, index_map, module_types, ModuleBay, ModuleType, Module + ) assert result["status"] == "failed" diff --git a/netbox_librenms_plugin/views/base/modules_view.py b/netbox_librenms_plugin/views/base/modules_view.py index 2964828f24..1bcb434d6f 100644 --- a/netbox_librenms_plugin/views/base/modules_view.py +++ b/netbox_librenms_plugin/views/base/modules_view.py @@ -366,7 +366,7 @@ def _get_sub_components(self, parent_idx, inventory_data): Returns list of (depth, item) tuples. """ results = [] - self._collect_descendants(parent_idx, inventory_data, depth=1, results=results, visited=set()) + self._collect_descendants(parent_idx, inventory_data, depth=1, results=results, visited={parent_idx}) return results def _collect_descendants(self, parent_idx, inventory_data, depth, results, visited=None): diff --git a/netbox_librenms_plugin/views/imports/actions.py b/netbox_librenms_plugin/views/imports/actions.py index 6df556a673..125b9f405a 100644 --- a/netbox_librenms_plugin/views/imports/actions.py +++ b/netbox_librenms_plugin/views/imports/actions.py @@ -1249,9 +1249,13 @@ def post(self, request, device_id): status=400, ) # Check that no other device already owns this ID on this server + # (both new namespaced format and legacy integer format) server_key = self.librenms_api.server_key conflict = ( - Device.objects.filter(**{f"custom_field_data__librenms_id__{server_key}": cf_locked}) + Device.objects.filter( + Q(**{f"custom_field_data__librenms_id__{server_key}": cf_locked}) + | Q(custom_field_data__librenms_id=cf_locked) + ) .exclude(pk=locked_device.pk) .exists() ) diff --git a/netbox_librenms_plugin/views/sync/modules.py b/netbox_librenms_plugin/views/sync/modules.py index f3256fd2dc..d47f732c34 100644 --- a/netbox_librenms_plugin/views/sync/modules.py +++ b/netbox_librenms_plugin/views/sync/modules.py @@ -14,7 +14,7 @@ ) -class InstallModuleView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, View): +class InstallModuleView(LibreNMSPermissionMixin, NetBoxObjectPermissionMixin, CacheMixin, View): """Install a NetBox Module into a ModuleBay from LibreNMS inventory data.""" def post(self, request, pk): @@ -55,7 +55,7 @@ def post(self, request, pk): module.full_clean() module.save() - cache.delete(f"librenms_inventory_device_{device.pk}") + cache.delete(self.get_cache_key(device, "inventory")) messages.success( request, f"Installed {module_type.model} in {module_bay.name} (serial: {serial or 'N/A'})." ) @@ -222,9 +222,12 @@ def _install_single(self, device, item, index_map, module_types, ModuleBay, Modu # Re-fetch module bays (parent install creates new child bays) bays = ModuleBay.objects.filter(device=device).select_related("installed_module__module_type") + # Pre-fetch all bay mappings once to avoid N+1 queries in _find_parent_module_id + bay_mappings = list(ModuleBayMapping.objects.all()) + # Determine if this item belongs under an installed module # by tracing its LibreNMS parent hierarchy to an installed item - parent_module_id = self._find_parent_module_id(item, index_map, device, ModuleBay) + parent_module_id = self._find_parent_module_id(item, index_map, bays, bay_mappings) if parent_module_id: bay_dict = {bay.name: bay for bay in bays if bay.module_id == parent_module_id} @@ -265,15 +268,31 @@ def _install_single(self, device, item, index_map, module_types, ModuleBay, Modu return {"status": "installed", "name": f"{matched_type.model} → {matched_bay.name}"} @staticmethod - def _find_parent_module_id(item, index_map, device, ModuleBay): + def _find_parent_module_id(item, index_map, device_bays, bay_mappings): """Find the NetBox module ID for the installed parent of this inventory item. Walks up the LibreNMS hierarchy to find an ancestor whose name matches an installed module bay on the device. + + Args: + item: The inventory item dict. + index_map: Dict mapping entPhysicalIndex to inventory item. + device_bays: Pre-fetched queryset/list of ModuleBay objects for the device. + bay_mappings: Pre-fetched list of all ModuleBayMapping objects. """ - from netbox_librenms_plugin.models import ModuleBayMapping current = item + # Build bay name->bay dict from pre-fetched bays for fast lookup + bay_by_name = {} + for bay in device_bays: + if bay.name not in bay_by_name: + bay_by_name[bay.name] = bay + # Build mapping dict keyed by librenms_name for fast lookup + mapping_by_name = {} + for m in bay_mappings: + if m.librenms_name not in mapping_by_name: + mapping_by_name[m.librenms_name] = m + for _ in range(10): # max depth guard parent_idx = current.get("entPhysicalContainedIn", 0) if not parent_idx or parent_idx not in index_map: @@ -283,24 +302,18 @@ def _find_parent_module_id(item, index_map, device, ModuleBay): parent_descr = parent.get("entPhysicalDescr", "") # Check if this parent matches an installed module bay on the device - device_bays = ModuleBay.objects.filter(device=device).select_related("installed_module") - for bay in device_bays: if hasattr(bay, "installed_module") and bay.installed_module: if bay.name == parent_name or (parent_descr and bay.name == parent_descr): return bay.installed_module.pk - # Also check ModuleBayMapping for indirect matches + # Also check ModuleBayMapping for indirect matches using pre-fetched data for name in [parent_name, parent_descr]: if not name: continue - mapping = ModuleBayMapping.objects.filter(librenms_name=name).first() + mapping = mapping_by_name.get(name) if mapping: - bay = ( - ModuleBay.objects.filter(device=device, name=mapping.netbox_bay_name) - .select_related("installed_module") - .first() - ) + bay = bay_by_name.get(mapping.netbox_bay_name) if bay and hasattr(bay, "installed_module") and bay.installed_module: return bay.installed_module.pk From bee4af211b686b146d8ee80f8716b757a0f33072 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 5 Mar 2026 15:14:48 +0100 Subject: [PATCH 20/28] chore: update pre-commit to ignore mkdocs.yml --- .github/pull_request_template.md | 2 +- .pre-commit-config.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index b641e68a71..99ff02374f 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -36,7 +36,7 @@ Delete items that don’t apply and describe briefly. 3. ## Risk Assessment -- Does this change affect existing users? +- Does this change affect existing users? - Could this cause unintended imports / updates? Explain briefly. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c51eb902da..5f71189fea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,5 +14,6 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml + exclude: mkdocs\.yml$ - id: check-added-large-files - id: check-merge-conflict From 865a39e63ea3c5a3ff20504e3c21fdf4ff371878 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 5 Mar 2026 15:15:21 +0100 Subject: [PATCH 21/28] chore: update pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f71189fea..acfaf7983b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.13 # Use the latest version from https://github.com/astral-sh/ruff-pre-commit/releases + rev: v0.15.4 # Use the latest version from https://github.com/astral-sh/ruff-pre-commit/releases hooks: # Run the linter - id: ruff-check From 9793c65e0278f701cf0cbf1e872e2750d90525a9 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 5 Mar 2026 15:24:29 +0100 Subject: [PATCH 22/28] chore: update dependabot/C901 --- .github/dependabot.yml | 4 ++++ pyproject.toml | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f6faee6938..5e142be68a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,3 +8,7 @@ updates: github-actions: patterns: - "*" + - package-ecosystem: "uv" + directory: "/" + schedule: + interval: "weekly" diff --git a/pyproject.toml b/pyproject.toml index 4620828506..316285ba8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,11 @@ addopts = "-v --tb=short" [tool.ruff] line-length = 120 +[tool.ruff.lint.mccabe] +#Flag errors (`C901`) whenever the complexity level exceeds 15. +#Rule not enforced - only to bump default 10 to 15 to allow for manual check +max-complexity = 15 + [tool.ruff.lint] # Follow NetBox conventions - ignore certain rules ignore = [ From 2b38f849da3b2fa109a2110dd23590c5c542e05d Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 5 Mar 2026 15:55:17 +0100 Subject: [PATCH 23/28] Fix second batch of code review findings - modules_view: guard against KeyError on missing entPhysicalIndex in index_map builds - modules_view: replace fixed range(10) ancestor traversal loop with visited-set while loop - modules_view: preload ModuleBayMapping in _build_context to eliminate N+1 queries in _match_module_bay - modules_view: update _lookup_regex_bay_mapping signature to accept preloaded list instead of querying DB - sync/modules: use require_all_permissions (HTML redirect) instead of JSON variant for form-based POST handlers - sync/modules: use ordered unique list (dict.fromkeys) instead of set for selected_indices to preserve install order - sync/modules: guard index_map build in InstallSelectedView against missing entPhysicalIndex - _module_sync_content.html: fix nested forms; move Install Selected into standalone form above table - librenms_sync.js: add initializeInstallSelectedForm() to collect checked rows before form submit - bulk_import: filter placeholder serials ('-', whitespace) from vc_domain member_serials - mock_librenms_server: add server_close() and thread join to stop() for proper cleanup - test_librenms_id: assert filter Q covers both JSON server-key and legacy integer paths - vm_operations: validate/convert device_id before VirtualMachine.objects.create - test_sync_modules: always import modules_view in test_install_module_view_not_in_base --- .../import_utils/bulk_import.py | 4 +- .../import_utils/vm_operations.py | 6 +- .../js/librenms_sync.js | 28 +++++++++ .../_module_sync_content.html | 13 ++-- .../tests/mock_librenms_server.py | 2 + .../tests/test_librenms_id.py | 6 ++ .../tests/test_sync_modules.py | 14 ++--- .../views/base/modules_view.py | 60 +++++++++++++------ netbox_librenms_plugin/views/sync/modules.py | 13 ++-- 9 files changed, 105 insertions(+), 41 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/bulk_import.py b/netbox_librenms_plugin/import_utils/bulk_import.py index c431c77153..92729413df 100644 --- a/netbox_librenms_plugin/import_utils/bulk_import.py +++ b/netbox_librenms_plugin/import_utils/bulk_import.py @@ -212,9 +212,9 @@ def bulk_import_devices_shared( # LibreNMS) share the same key and VC creation is triggered only once. # Fall back to device_id when no member serials are available. member_serials = sorted( - str(m.get("serial")) + serial for m in vc_data.get("members", []) - if m.get("serial") is not None and m.get("serial") != "" + if (serial := str(m.get("serial") or "").strip()) and serial != "-" ) vc_domain = ( f"librenms-stack-{','.join(member_serials)}" if member_serials else f"librenms-{device_id}" diff --git a/netbox_librenms_plugin/import_utils/vm_operations.py b/netbox_librenms_plugin/import_utils/vm_operations.py index d97a3ba669..dfc6b260f5 100644 --- a/netbox_librenms_plugin/import_utils/vm_operations.py +++ b/netbox_librenms_plugin/import_utils/vm_operations.py @@ -54,6 +54,10 @@ def create_vm_from_librenms( # Generate import timestamp comment import_time = timezone.now().strftime("%Y-%m-%d %H:%M:%S %Z") + # Validate device_id before creating the VM so a missing/invalid value + # never leaves a VM without a librenms_id (partial persistence). + librenms_device_id = int(libre_device["device_id"]) + # Create the VM with librenms_id custom field vm = VirtualMachine.objects.create( name=vm_name, @@ -65,7 +69,7 @@ def create_vm_from_librenms( from ..utils import set_librenms_device_id - set_librenms_device_id(vm, int(libre_device["device_id"]), server_key) + set_librenms_device_id(vm, librenms_device_id, server_key) vm.save() logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") diff --git a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js index 0b42112b24..f43ce20672 100644 --- a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js +++ b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js @@ -1404,6 +1404,33 @@ function initializeSyncFormSpinners() { * Initialize all sync page functionality. * Called on DOMContentLoaded and after HTMX content swaps. */ +/** + * Wire the "Install Selected" form to collect checked module-table rows before submit. + * The form is separate from the table (to avoid nested forms), so we copy the + * selected checkbox values into hidden inputs just before the form is submitted. + */ +function initializeInstallSelectedForm() { + const form = document.getElementById('install-selected-form'); + if (!form) return; + + form.addEventListener('submit', function () { + // Remove any previously-injected hidden inputs to avoid duplicates + form.querySelectorAll('input[data-injected-select]').forEach(el => el.remove()); + + const table = document.getElementById('librenms-module-table'); + if (!table) return; + + table.querySelectorAll('input[name="select"]:checked').forEach(cb => { + const hidden = document.createElement('input'); + hidden.type = 'hidden'; + hidden.name = 'select'; + hidden.value = cb.value; + hidden.dataset.injectedSelect = '1'; + form.appendChild(hidden); + }); + }); +} + function initializeScripts() { initializeCheckboxes(); initializeVCMemberSelect(); @@ -1420,6 +1447,7 @@ function initializeScripts() { initializeNetBoxOnlyInterfaces(); initializeSyncFormSpinners(); initializeVlanSyncGroupSelects(); + initializeInstallSelectedForm(); } diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html index a177ba7d05..4424c53f43 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html @@ -16,7 +16,8 @@ {% endif %}
-
{% csrf_token %}
@@ -24,12 +25,12 @@ Install Selected
-
- {% include 'netbox_librenms_plugin/inc/paginator.html' with table=module_sync.table %} - {% include 'inc/table.html' with table=module_sync.table %} - {% include 'netbox_librenms_plugin/inc/paginator.html' with table=module_sync.table %} -
+
+ {% include 'netbox_librenms_plugin/inc/paginator.html' with table=module_sync.table %} + {% include 'inc/table.html' with table=module_sync.table %} + {% include 'netbox_librenms_plugin/inc/paginator.html' with table=module_sync.table %} +
{% else %}
diff --git a/netbox_librenms_plugin/tests/mock_librenms_server.py b/netbox_librenms_plugin/tests/mock_librenms_server.py index 51838e52a0..9ca8b1ec6e 100644 --- a/netbox_librenms_plugin/tests/mock_librenms_server.py +++ b/netbox_librenms_plugin/tests/mock_librenms_server.py @@ -69,6 +69,8 @@ def start(self): def stop(self): self._server.shutdown() + self._server.server_close() + self._thread.join(timeout=5) # ------- default LibreNMS-shaped responses ------- diff --git a/netbox_librenms_plugin/tests/test_librenms_id.py b/netbox_librenms_plugin/tests/test_librenms_id.py index bed6c5497e..d5a5b4cc99 100644 --- a/netbox_librenms_plugin/tests/test_librenms_id.py +++ b/netbox_librenms_plugin/tests/test_librenms_id.py @@ -80,6 +80,12 @@ def test_queries_server_key_and_legacy_integer(self): find_by_librenms_id(mock_model, 42, "default") mock_model.objects.filter.assert_called_once() + # Verify the Q argument covers both the JSON server-key path and legacy integer path. + call_args = mock_model.objects.filter.call_args + q_arg = call_args[0][0] + q_str = str(q_arg) + assert "librenms_id__default" in q_str, "Expected JSON-scoped server_key lookup in filter" + assert "librenms_id" in q_str, "Expected legacy integer lookup in filter" def test_returns_first_matching_object(self): from netbox_librenms_plugin.utils import find_by_librenms_id diff --git a/netbox_librenms_plugin/tests/test_sync_modules.py b/netbox_librenms_plugin/tests/test_sync_modules.py index 0d0f4c8062..18aa5fb8e6 100644 --- a/netbox_librenms_plugin/tests/test_sync_modules.py +++ b/netbox_librenms_plugin/tests/test_sync_modules.py @@ -162,14 +162,12 @@ def test_has_netbox_object_permission_mixin(self): def test_install_module_view_not_in_base(self): """InstallModuleView must NOT be defined in views/base anymore.""" - import sys - - # Reload to avoid cached state - if "netbox_librenms_plugin.views.base.modules_view" in sys.modules: - mod = sys.modules["netbox_librenms_plugin.views.base.modules_view"] - assert not hasattr(mod, "InstallModuleView"), ( - "InstallModuleView must have been moved out of views/base/modules_view.py" - ) + import importlib + + mod = importlib.import_module("netbox_librenms_plugin.views.base.modules_view") + assert not hasattr(mod, "InstallModuleView"), ( + "InstallModuleView must have been moved out of views/base/modules_view.py" + ) class TestInstallBranchViewWiring: diff --git a/netbox_librenms_plugin/views/base/modules_view.py b/netbox_librenms_plugin/views/base/modules_view.py index 1bcb434d6f..b674d22f45 100644 --- a/netbox_librenms_plugin/views/base/modules_view.py +++ b/netbox_librenms_plugin/views/base/modules_view.py @@ -95,11 +95,19 @@ def get_context_data(self, request, obj): def _build_context(self, request, obj, inventory_data): """Build context with matched inventory items and table.""" # Build a lookup of all inventory items by index for parent resolution - index_map = {item["entPhysicalIndex"]: item for item in inventory_data} + # Skip items with missing entPhysicalIndex to avoid KeyError on malformed data. + index_map = {idx: item for item in inventory_data if (idx := item.get("entPhysicalIndex")) is not None} # Store manufacturer for normalization rules in _build_row self._device_manufacturer = getattr(getattr(obj, "device_type", None), "manufacturer", None) + # Preload all ModuleBayMapping rows once to avoid N+1 queries in _match_module_bay. + from netbox_librenms_plugin.models import ModuleBayMapping + + all_bay_mappings = list(ModuleBayMapping.objects.all()) + self._exact_bay_mappings = [m for m in all_bay_mappings if not m.is_regex] + self._regex_bay_mappings = [m for m in all_bay_mappings if m.is_regex] + # Get NetBox module bays and modules for this device device_bays, module_scoped_bays = self._get_module_bays(obj) module_types = self._get_module_types() @@ -128,9 +136,9 @@ def _build_context(self, request, obj, inventory_data): # real modules — skip them so children can be top-level items. is_descendant = False current_idx = item.get("entPhysicalContainedIn", 0) - for _ in range(10): - if not current_idx or current_idx not in index_map: - break + visited_ancestors = set() + while current_idx and current_idx in index_map and current_idx not in visited_ancestors: + visited_ancestors.add(current_idx) ancestor = index_map[current_idx] anc_class = ancestor.get("entPhysicalClass") if anc_class in INVENTORY_CLASSES: @@ -470,8 +478,6 @@ def _match_module_bay(self, item, index_map, module_bays): """ import re - from netbox_librenms_plugin.models import ModuleBayMapping - parent_name = self._find_parent_container_name(item, index_map) item_name = item.get("entPhysicalName", "") item_descr = item.get("entPhysicalDescr", "") @@ -480,22 +486,39 @@ def _match_module_bay(self, item, index_map, module_bays): # Build candidate names: parent, item name, item description candidate_names = [n for n in [parent_name, item_name, item_descr] if n] + # Use preloaded exact mappings (set in _build_context to avoid N+1 queries). + exact_mappings = getattr(self, "_exact_bay_mappings", None) + if exact_mappings is None: + from netbox_librenms_plugin.models import ModuleBayMapping + + exact_mappings = list(ModuleBayMapping.objects.filter(is_regex=False)) + # Check ModuleBayMapping table for each candidate (exact match) for name in candidate_names: - filters = {"librenms_name": name, "is_regex": False} if phys_class: - mapping = ModuleBayMapping.objects.filter(**filters, librenms_class=phys_class).first() + mapping = next( + (m for m in exact_mappings if m.librenms_name == name and m.librenms_class == phys_class), None + ) if not mapping: - mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + mapping = next( + (m for m in exact_mappings if m.librenms_name == name and m.librenms_class == ""), None + ) else: - mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + mapping = next((m for m in exact_mappings if m.librenms_name == name and m.librenms_class == ""), None) if mapping and mapping.netbox_bay_name in module_bays: return module_bays[mapping.netbox_bay_name] + # Use preloaded regex mappings. + regex_mappings = getattr(self, "_regex_bay_mappings", None) + if regex_mappings is None: + from netbox_librenms_plugin.models import ModuleBayMapping + + regex_mappings = list(ModuleBayMapping.objects.filter(is_regex=True)) + # Regex pattern matching on all candidate names for name in candidate_names: - bay = self._lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping) + bay = self._lookup_regex_bay_mapping(re, name, phys_class, module_bays, regex_mappings) if bay and self._fpc_slot_matches(name, bay): return bay @@ -537,20 +560,21 @@ def _fpc_slot_matches(candidate_name, bay): return parent_bay.position == expected_fpc @staticmethod - def _lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping): + def _lookup_regex_bay_mapping(re, name, phys_class, module_bays, regex_mappings): """Try regex ModuleBayMapping patterns against a name. + ``regex_mappings`` is a pre-filtered list of is_regex=True ModuleBayMapping + objects (passed in from the caller to avoid per-item DB queries). + Returns matched module bay or None. """ - regex_filters = {"is_regex": True} + # Filter preloaded list by class (exact class match or empty-class fallback) if phys_class: - regex_mappings = list(ModuleBayMapping.objects.filter(**regex_filters, librenms_class=phys_class)) + list( - ModuleBayMapping.objects.filter(**regex_filters, librenms_class="") - ) + candidates = [m for m in regex_mappings if m.librenms_class == phys_class or m.librenms_class == ""] else: - regex_mappings = list(ModuleBayMapping.objects.filter(**regex_filters, librenms_class="")) + candidates = [m for m in regex_mappings if m.librenms_class == ""] - for mapping in regex_mappings: + for mapping in candidates: try: match = re.fullmatch(mapping.librenms_name, name) except re.error: diff --git a/netbox_librenms_plugin/views/sync/modules.py b/netbox_librenms_plugin/views/sync/modules.py index d47f732c34..ba8291f015 100644 --- a/netbox_librenms_plugin/views/sync/modules.py +++ b/netbox_librenms_plugin/views/sync/modules.py @@ -21,7 +21,7 @@ def post(self, request, pk): from dcim.models import Device, Module, ModuleBay, ModuleType self.required_object_permissions = {"POST": [("add", Module)]} - if error := self.require_all_permissions_json("POST"): + if error := self.require_all_permissions("POST"): return error device = get_object_or_404(Device, pk=pk) @@ -73,7 +73,7 @@ def post(self, request, pk): from dcim.models import Device, Module, ModuleBay, ModuleType self.required_object_permissions = {"POST": [("add", Module)]} - if error := self.require_all_permissions_json("POST"): + if error := self.require_all_permissions("POST"): return error device = get_object_or_404(Device, pk=pk) @@ -382,7 +382,7 @@ def post(self, request, pk): from dcim.models import Device, Module, ModuleBay, ModuleType self.required_object_permissions = {"POST": [("add", Module)]} - if error := self.require_all_permissions_json("POST"): + if error := self.require_all_permissions("POST"): return error device = get_object_or_404(Device, pk=pk) @@ -400,14 +400,15 @@ def post(self, request, pk): return redirect(f"{sync_url}?tab=modules#librenms-module-table") try: - selected_set = {int(i) for i in selected_indices} + # Use dict.fromkeys to preserve order while deduplicating + selected_list = list(dict.fromkeys(int(i) for i in selected_indices)) except ValueError: messages.error(request, "Invalid selection.") sync_url = reverse("plugins:netbox_librenms_plugin:device_librenms_sync", kwargs={"pk": pk}) return redirect(f"{sync_url}?tab=modules#librenms-module-table") - index_map = {item["entPhysicalIndex"]: item for item in cached_data} - items = [index_map[idx] for idx in selected_set if idx in index_map] + index_map = {idx: item for item in cached_data if (idx := item.get("entPhysicalIndex")) is not None} + items = [index_map[idx] for idx in selected_list if idx in index_map] if not items: messages.warning(request, "None of the selected indices matched cached inventory.") From d3f917a087d7e6149eaad1517f99ea86d04337cf Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 5 Mar 2026 16:18:17 +0100 Subject: [PATCH 24/28] fix: module tab checkboxes and Cisco 8201 transceiver visibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug 1: ToggleColumn in LibreNMSModuleTable lacked an accessor, so the resolved cell value was always '' (empty_values) and render() was never called → no per-row checkboxes. Fix: add accessor='ent_physical_index' which is present on every row dict produced by _build_row. Bug 2: On Cisco 8201-style inventory the top-level chassis module (idx 1) is wrapped in a container that has model='N/A'. The ancestor-walk skipped containers only when their model was truly empty (not anc_model), but 'N/A' is truthy, so the container was treated as a real ancestor and the chassis module was erroneously marked is_descendant=True → excluded from top_items → no rows and no transceivers. Fix: extend the skip condition to check anc_model in _GENERIC_CONTAINER_MODELS instead of not anc_model. Add regression tests for both bugs: - TestToggleColumnAccessor: asserts selection column has correct accessor and that a record with ent_physical_index produces a non-empty checkbox. - TestAncestorWalkGenericContainerModel: asserts items under N/A containers appear as top-level items, and that real parent/child relationships are still handled correctly. --- netbox_librenms_plugin/tables/modules.py | 1 + .../tests/test_sync_modules.py | 157 ++++++++++++++++++ .../views/base/modules_view.py | 4 +- 3 files changed, 160 insertions(+), 2 deletions(-) diff --git a/netbox_librenms_plugin/tables/modules.py b/netbox_librenms_plugin/tables/modules.py index caaad905f1..87788eb19a 100644 --- a/netbox_librenms_plugin/tables/modules.py +++ b/netbox_librenms_plugin/tables/modules.py @@ -13,6 +13,7 @@ class LibreNMSModuleTable(tables.Table): selection = ToggleColumn( orderable=False, visible=True, + accessor="ent_physical_index", attrs={"td": {"data-col": "selection"}, "input": {"name": "select"}}, ) name = tables.Column( diff --git a/netbox_librenms_plugin/tests/test_sync_modules.py b/netbox_librenms_plugin/tests/test_sync_modules.py index 18aa5fb8e6..4639be1e45 100644 --- a/netbox_librenms_plugin/tests/test_sync_modules.py +++ b/netbox_librenms_plugin/tests/test_sync_modules.py @@ -759,3 +759,160 @@ def noop_atomic(): ) assert result["status"] == "failed" + + +# --------------------------------------------------------------------------- +# Regression: ToggleColumn accessor for per-row checkboxes +# --------------------------------------------------------------------------- + + +class TestToggleColumnAccessor: + """ToggleColumn must have accessor='ent_physical_index' so per-row checkboxes render.""" + + def test_selection_column_has_correct_accessor(self): + """Regression: without accessor='ent_physical_index' checkboxes are empty.""" + from netbox_librenms_plugin.tables.modules import LibreNMSModuleTable + + col = LibreNMSModuleTable.base_columns["selection"] + assert col.accessor == "ent_physical_index", ( + "ToggleColumn must use accessor='ent_physical_index'; " + "otherwise the column value resolves to '' and render() is never called" + ) + + def test_selection_column_renders_checkbox_for_record_with_index(self): + """Per-row checkbox renders when ent_physical_index is present in record.""" + from netbox_librenms_plugin.tables.modules import LibreNMSModuleTable + + record = { + "ent_physical_index": 42, + "name": "Slot 1", + "model": "WS-X4748", + "depth": 0, + } + table = LibreNMSModuleTable([record]) + rows = list(table.rows) + assert len(rows) == 1 + # The cell value for 'selection' should be 42 (ent_physical_index), not '' + cell_val = rows[0].get_cell("selection") + assert str(cell_val) != "", "Checkbox cell must not be empty for a record with ent_physical_index" + + +# --------------------------------------------------------------------------- +# Regression: ancestor walk skips containers with N/A model (Cisco 8201 style) +# --------------------------------------------------------------------------- + + +class TestAncestorWalkGenericContainerModel: + """Top-level items under containers with 'N/A' model should not be excluded.""" + + def _run_top_items(self, inventory_data): + from netbox_librenms_plugin.views.base.modules_view import INVENTORY_CLASSES, _GENERIC_CONTAINER_MODELS + + idx_map = { + item["entPhysicalIndex"]: item for item in inventory_data if item.get("entPhysicalIndex") is not None + } + top_items = [] + for item in inventory_data: + phys_class = item.get("entPhysicalClass") + if phys_class not in INVENTORY_CLASSES: + continue + model = (item.get("entPhysicalModelName") or "").strip() + if phys_class == "container" and model in _GENERIC_CONTAINER_MODELS: + continue + if model and model in _GENERIC_CONTAINER_MODELS: + continue + is_descendant = False + current_idx = item.get("entPhysicalContainedIn", 0) + visited_ancestors = set() + while current_idx and current_idx in idx_map and current_idx not in visited_ancestors: + visited_ancestors.add(current_idx) + ancestor = idx_map[current_idx] + anc_class = ancestor.get("entPhysicalClass") + if anc_class in INVENTORY_CLASSES: + anc_model = (ancestor.get("entPhysicalModelName") or "").strip() + if anc_class == "container" and anc_model in _GENERIC_CONTAINER_MODELS: + current_idx = ancestor.get("entPhysicalContainedIn", 0) + continue + is_descendant = True + break + current_idx = ancestor.get("entPhysicalContainedIn", 0) + if is_descendant: + continue + top_items.append(item) + return top_items + + def test_item_under_container_with_na_model_is_top_level(self): + """Module under a container with model='N/A' must appear as top-level item.""" + inventory = [ + # chassis (not in INVENTORY_CLASSES, so ignored in ancestor walk) + { + "entPhysicalIndex": 9000, + "entPhysicalClass": "chassis", + "entPhysicalModelName": "8201-SYS", + "entPhysicalContainedIn": 0, + }, + # container with model='N/A' inside chassis — generic slot + { + "entPhysicalIndex": 8000, + "entPhysicalClass": "container", + "entPhysicalModelName": "N/A", + "entPhysicalContainedIn": 9000, + }, + # real module inside the N/A container — should be top-level + { + "entPhysicalIndex": 1, + "entPhysicalClass": "module", + "entPhysicalModelName": "8201-SYS", + "entPhysicalContainedIn": 8000, + }, + ] + top = self._run_top_items(inventory) + indices = [i["entPhysicalIndex"] for i in top] + assert 1 in indices, "Module inside N/A container must be a top-level item (Cisco 8201 regression)" + + def test_item_under_container_with_empty_model_is_top_level(self): + """Legacy: module under container with empty model still works.""" + inventory = [ + { + "entPhysicalIndex": 9000, + "entPhysicalClass": "chassis", + "entPhysicalModelName": "8201-SYS", + "entPhysicalContainedIn": 0, + }, + { + "entPhysicalIndex": 8000, + "entPhysicalClass": "container", + "entPhysicalModelName": "", + "entPhysicalContainedIn": 9000, + }, + { + "entPhysicalIndex": 1, + "entPhysicalClass": "module", + "entPhysicalModelName": "8201-SYS", + "entPhysicalContainedIn": 8000, + }, + ] + top = self._run_top_items(inventory) + indices = [i["entPhysicalIndex"] for i in top] + assert 1 in indices + + def test_item_under_real_module_is_excluded(self): + """Module inside another real (non-generic) module stays a descendant.""" + inventory = [ + { + "entPhysicalIndex": 1, + "entPhysicalClass": "module", + "entPhysicalModelName": "PARENT-MODULE", + "entPhysicalContainedIn": 0, + }, + { + "entPhysicalIndex": 2, + "entPhysicalClass": "module", + "entPhysicalModelName": "CHILD-MODULE", + "entPhysicalContainedIn": 1, + }, + ] + top = self._run_top_items(inventory) + indices = [i["entPhysicalIndex"] for i in top] + assert 1 in indices + assert 2 not in indices, "Child module under real parent must remain a descendant" diff --git a/netbox_librenms_plugin/views/base/modules_view.py b/netbox_librenms_plugin/views/base/modules_view.py index b674d22f45..43d8353f77 100644 --- a/netbox_librenms_plugin/views/base/modules_view.py +++ b/netbox_librenms_plugin/views/base/modules_view.py @@ -143,8 +143,8 @@ def _build_context(self, request, obj, inventory_data): anc_class = ancestor.get("entPhysicalClass") if anc_class in INVENTORY_CLASSES: anc_model = (ancestor.get("entPhysicalModelName") or "").strip() - # Empty-model containers are just physical slot representations - if anc_class == "container" and not anc_model: + # Containers with generic/empty models are physical slot representations + if anc_class == "container" and anc_model in _GENERIC_CONTAINER_MODELS: current_idx = ancestor.get("entPhysicalContainedIn", 0) continue is_descendant = True From f6ebcf3575a02bc63b585e9dcbf26baae3ca6782 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 5 Mar 2026 16:50:08 +0100 Subject: [PATCH 25/28] =?UTF-8?q?fix:=207=20code-review=20findings=20?= =?UTF-8?q?=E2=80=94=20correctness=20and=20robustness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sync/modules.py: _match_bay passed ModuleBayMapping class to _lookup_regex_bay_mapping instead of the preloaded is_regex=True list; fix by pre-fetching regex mappings before the loop and passing the list. Also guard InstallBranchView index_map build against missing entPhysicalIndex (mirrors the existing guard in InstallSelectedView). modules_view.py: replace fixed-bound 'for _ in range(5)' loop in _match_bay_by_position with a visited-set while loop so deeper containment chains are handled correctly and cycles are detected. Also guard 'item["entPhysicalIndex"]' direct access with .get() + continue so malformed top_items rows are skipped safely. vm_operations.py: wrap VirtualMachine.objects.create and set_librenms_device_id in a single transaction.atomic() block so a failure during ID assignment never leaves a VM without a LibreNMS mapping. Update test fixture with an autouse _patch_atomic to keep tests DB-free. librenms_sync.js: initializeInstallSelectedForm previously added a new submit listener on every HTMX swap. Guard with form.dataset.installInit flag so the listener is registered exactly once per form element. mock_librenms_server.py: add thread.is_alive() warning after join() so test failures from unreleased sockets are surfaced immediately. Also add ifMtu/ifVlan/ifTrunk defaults to ports_response to match the real LibreNMS /ports API shape. --- .../import_utils/vm_operations.py | 24 +++++++------ .../js/librenms_sync.js | 36 +++++++++++-------- .../tests/mock_librenms_server.py | 12 +++++++ .../tests/test_vm_operations.py | 12 +++++++ .../views/base/modules_view.py | 14 +++++--- netbox_librenms_plugin/views/sync/modules.py | 7 ++-- 6 files changed, 71 insertions(+), 34 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/vm_operations.py b/netbox_librenms_plugin/import_utils/vm_operations.py index dfc6b260f5..4017cc97f7 100644 --- a/netbox_librenms_plugin/import_utils/vm_operations.py +++ b/netbox_librenms_plugin/import_utils/vm_operations.py @@ -3,6 +3,7 @@ import logging from dcim.models import DeviceRole +from django.db import transaction from django.utils import timezone from virtualization.models import Cluster @@ -58,19 +59,20 @@ def create_vm_from_librenms( # never leaves a VM without a librenms_id (partial persistence). librenms_device_id = int(libre_device["device_id"]) - # Create the VM with librenms_id custom field - vm = VirtualMachine.objects.create( - name=vm_name, - cluster=cluster, - role=role, # Optional VM role - platform=platform, - comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", - ) - from ..utils import set_librenms_device_id - set_librenms_device_id(vm, librenms_device_id, server_key) - vm.save() + # Create the VM and assign its LibreNMS ID atomically so a failure in + # set_librenms_device_id never leaves a VM without a mapping. + with transaction.atomic(): + vm = VirtualMachine.objects.create( + name=vm_name, + cluster=cluster, + role=role, # Optional VM role + platform=platform, + comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", + ) + set_librenms_device_id(vm, librenms_device_id, server_key) + vm.save() logger.info(f"Created VM {vm.name} (ID: {vm.pk}) from LibreNMS device {libre_device['device_id']}") return vm diff --git a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js index f43ce20672..90e3943dd5 100644 --- a/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js +++ b/netbox_librenms_plugin/static/netbox_librenms_plugin/js/librenms_sync.js @@ -1408,29 +1408,35 @@ function initializeSyncFormSpinners() { * Wire the "Install Selected" form to collect checked module-table rows before submit. * The form is separate from the table (to avoid nested forms), so we copy the * selected checkbox values into hidden inputs just before the form is submitted. + * Guard against duplicate listeners on repeated HTMX swaps via a data attribute. */ -function initializeInstallSelectedForm() { +function handleInstallSelectedSubmit() { + // Remove any previously-injected hidden inputs to avoid duplicates const form = document.getElementById('install-selected-form'); if (!form) return; + form.querySelectorAll('input[data-injected-select]').forEach(el => el.remove()); - form.addEventListener('submit', function () { - // Remove any previously-injected hidden inputs to avoid duplicates - form.querySelectorAll('input[data-injected-select]').forEach(el => el.remove()); - - const table = document.getElementById('librenms-module-table'); - if (!table) return; + const table = document.getElementById('librenms-module-table'); + if (!table) return; - table.querySelectorAll('input[name="select"]:checked').forEach(cb => { - const hidden = document.createElement('input'); - hidden.type = 'hidden'; - hidden.name = 'select'; - hidden.value = cb.value; - hidden.dataset.injectedSelect = '1'; - form.appendChild(hidden); - }); + table.querySelectorAll('input[name="select"]:checked').forEach(cb => { + const hidden = document.createElement('input'); + hidden.type = 'hidden'; + hidden.name = 'select'; + hidden.value = cb.value; + hidden.dataset.injectedSelect = '1'; + form.appendChild(hidden); }); } +function initializeInstallSelectedForm() { + const form = document.getElementById('install-selected-form'); + if (!form) return; + if (form.dataset.installInit) return; + form.dataset.installInit = 'true'; + form.addEventListener('submit', handleInstallSelectedSubmit); +} + function initializeScripts() { initializeCheckboxes(); initializeVCMemberSelect(); diff --git a/netbox_librenms_plugin/tests/mock_librenms_server.py b/netbox_librenms_plugin/tests/mock_librenms_server.py index 9ca8b1ec6e..652c73370b 100644 --- a/netbox_librenms_plugin/tests/mock_librenms_server.py +++ b/netbox_librenms_plugin/tests/mock_librenms_server.py @@ -71,6 +71,15 @@ def stop(self): self._server.shutdown() self._server.server_close() self._thread.join(timeout=5) + if self._thread.is_alive(): + import warnings + + warnings.warn( + f"MockLibreNMSServer thread {self._thread.ident} did not exit within 5 s; " + "socket may not be fully released", + ResourceWarning, + stacklevel=2, + ) # ------- default LibreNMS-shaped responses ------- @@ -117,6 +126,9 @@ def ports_response(self, device_id: int = 1, ports=None): "ifAdminStatus": "up", "ifAlias": "uplink", "ifPhysAddress": "aa:bb:cc:dd:ee:01", + "ifMtu": 1500, + "ifVlan": 1, + "ifTrunk": 0, } ] self.register(f"/api/v0/devices/{device_id}/ports", {"status": "ok", "ports": ports}) diff --git a/netbox_librenms_plugin/tests/test_vm_operations.py b/netbox_librenms_plugin/tests/test_vm_operations.py index 743aa17992..76fa870445 100644 --- a/netbox_librenms_plugin/tests/test_vm_operations.py +++ b/netbox_librenms_plugin/tests/test_vm_operations.py @@ -11,6 +11,18 @@ class TestCreateVmFromLibrenms: """Tests for create_vm_from_librenms function.""" + @pytest.fixture(autouse=True) + def _patch_atomic(self): + """transaction.atomic() is a no-op; tests mock all DB interactions.""" + from contextlib import contextmanager + + @contextmanager + def noop_atomic(): + yield + + with patch("netbox_librenms_plugin.import_utils.vm_operations.transaction.atomic", noop_atomic): + yield + def test_success_with_computed_name(self): """VM is created using pre-computed _computed_name when present.""" from netbox_librenms_plugin.import_utils.vm_operations import create_vm_from_librenms diff --git a/netbox_librenms_plugin/views/base/modules_view.py b/netbox_librenms_plugin/views/base/modules_view.py index 43d8353f77..9d1cd8f059 100644 --- a/netbox_librenms_plugin/views/base/modules_view.py +++ b/netbox_librenms_plugin/views/base/modules_view.py @@ -198,7 +198,10 @@ def _build_context(self, request, obj, inventory_data): # Find sub-components with a model name (transceivers, converters, etc.) # Track bay scope per depth level so nested modules use correct bays bays_by_depth = {0: child_bays} - sub_items = self._get_sub_components(item["entPhysicalIndex"], inventory_data) + parent_idx = item.get("entPhysicalIndex") + if parent_idx is None: + continue + sub_items = self._get_sub_components(parent_idx, inventory_data) for depth, sub_item in sub_items: scope_bays = bays_by_depth.get(depth, child_bays) sub_row = self._build_row(sub_item, index_map, scope_bays, module_types, depth=depth) @@ -595,12 +598,13 @@ def _match_bay_by_position(item, index_map, module_bays): nearest ancestor with a model, count which container slot the item occupies, and match to the bay by number (e.g., SFP 1, SFP 2). """ - # Walk up through modelless containers to find the parent with a model + # Walk up through modelless containers to find the parent with a model. + # Use a visited set to detect cycles and avoid infinite loops. current_idx = item.get("entPhysicalContainedIn", 0) container_idx = None - for _ in range(5): - if not current_idx or current_idx not in index_map: - return None + visited = set() + while current_idx and current_idx in index_map and current_idx not in visited: + visited.add(current_idx) ancestor = index_map[current_idx] model = (ancestor.get("entPhysicalModelName") or "").strip() if model: diff --git a/netbox_librenms_plugin/views/sync/modules.py b/netbox_librenms_plugin/views/sync/modules.py index ba8291f015..2f90154b83 100644 --- a/netbox_librenms_plugin/views/sync/modules.py +++ b/netbox_librenms_plugin/views/sync/modules.py @@ -99,7 +99,7 @@ def post(self, request, pk): return redirect(f"{sync_url}?tab=modules#librenms-module-table") # Build index map and collect the branch to install - index_map = {item["entPhysicalIndex"]: item for item in cached_data} + index_map = {idx: item for item in cached_data if (idx := item.get("entPhysicalIndex")) is not None} branch_items = self._collect_branch(parent_index, cached_data) if not branch_items: @@ -354,9 +354,10 @@ def _match_bay(item, index_map, module_bays, ModuleBayMapping): if mapping and mapping.netbox_bay_name in module_bays: return module_bays[mapping.netbox_bay_name] - # Regex pattern matching on all candidate names + # Regex pattern matching on all candidate names (preload once to avoid N+1) + regex_mappings = list(ModuleBayMapping.objects.filter(is_regex=True)) for name in candidate_names: - bay = BaseModuleTableView._lookup_regex_bay_mapping(re, name, phys_class, module_bays, ModuleBayMapping) + bay = BaseModuleTableView._lookup_regex_bay_mapping(re, name, phys_class, module_bays, regex_mappings) if bay: return bay From 714b7309a2617c6d9efaf538d555ac1a0cafc9d3 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 5 Mar 2026 17:18:18 +0100 Subject: [PATCH 26/28] Fix 4th batch code review findings: parent_idx collision, N+1 mappings, hierarchy traversal, .get() guards - modules_view.py: rename parent_idx (row index) to parent_row_idx to eliminate variable collision with parent_ent_idx (entity physical index). table_data was being indexed with an ENTITY-MIB index instead of the table row index, causing potential IndexError or wrong-row mutations. - modules_view.py: guard inv_by_index dict comprehension with .get() so malformed rows missing entPhysicalIndex don't abort _merge_transceiver_data. - modules_view.py/_collect_descendants: guard child_idx with .get() + skip None. - sync/modules.py/_collect_branch: use .get('entPhysicalIndex') in parent lookup. - sync/modules.py/_collect_children: guard child_idx with .get() + continue. - sync/modules.py/_find_parent_module_id: replace for _ in range(10) with visited-set while loop to detect cycles and avoid depth-limited misses. - sync/modules.py/_match_bay: walk up full containment hierarchy for parent_name. - sync/modules.py/_match_bay: accept preloaded exact/regex mappings, replace N+1 ORM queries per candidate with in-memory dict lookup. - sync/modules.py/_install_single: accept optional exact/regex_mappings params. - InstallBranchView/InstallSelectedView: preload ModuleBayMapping once per request. - test_vm_operations.py: assert api.server_key is forwarded to create_vm_from_librenms. --- .../tests/test_vm_operations.py | 7 +- .../views/base/modules_view.py | 20 ++-- netbox_librenms_plugin/views/sync/modules.py | 108 ++++++++++++++---- 3 files changed, 100 insertions(+), 35 deletions(-) diff --git a/netbox_librenms_plugin/tests/test_vm_operations.py b/netbox_librenms_plugin/tests/test_vm_operations.py index 76fa870445..9a6e3da2e0 100644 --- a/netbox_librenms_plugin/tests/test_vm_operations.py +++ b/netbox_librenms_plugin/tests/test_vm_operations.py @@ -288,6 +288,8 @@ def test_success_path_vm_created(self): mock_vm = MagicMock() mock_vm.name = "new-vm" + mock_create_vm = MagicMock(return_value=mock_vm) + with ( patch("netbox_librenms_plugin.import_utils.vm_operations.require_permissions"), patch( @@ -304,7 +306,7 @@ def test_success_path_vm_created(self): ), patch( "netbox_librenms_plugin.import_utils.vm_operations.create_vm_from_librenms", - return_value=mock_vm, + mock_create_vm, ), patch("netbox_librenms_plugin.import_utils.vm_operations.Cluster"), patch("netbox_librenms_plugin.import_utils.vm_operations.DeviceRole"), @@ -318,6 +320,9 @@ def test_success_path_vm_created(self): assert result["success"][0]["device"] == mock_vm assert len(result["failed"]) == 0 assert len(result["skipped"]) == 0 + # Verify api.server_key is forwarded to create_vm_from_librenms + call_kwargs = mock_create_vm.call_args[1] + assert call_kwargs.get("server_key") == mock_api.server_key def test_cluster_assignment_applied(self): """apply_cluster_to_validation is called when cluster_id is provided and found.""" diff --git a/netbox_librenms_plugin/views/base/modules_view.py b/netbox_librenms_plugin/views/base/modules_view.py index 9d1cd8f059..2058065132 100644 --- a/netbox_librenms_plugin/views/base/modules_view.py +++ b/netbox_librenms_plugin/views/base/modules_view.py @@ -170,7 +170,7 @@ def _build_context(self, request, obj, inventory_data): # that share the same name as a device bay. item_bays = all_bays if item.get("_from_transceiver_api") else device_bays row = self._build_row(item, index_map, item_bays, module_types, depth=0) - parent_idx = len(table_data) + parent_row_idx = len(table_data) table_data.append(row) # Determine which bays sub-components should match against: @@ -198,10 +198,10 @@ def _build_context(self, request, obj, inventory_data): # Find sub-components with a model name (transceivers, converters, etc.) # Track bay scope per depth level so nested modules use correct bays bays_by_depth = {0: child_bays} - parent_idx = item.get("entPhysicalIndex") - if parent_idx is None: + parent_ent_idx = item.get("entPhysicalIndex") + if parent_ent_idx is None: continue - sub_items = self._get_sub_components(parent_idx, inventory_data) + sub_items = self._get_sub_components(parent_ent_idx, inventory_data) for depth, sub_item in sub_items: scope_bays = bays_by_depth.get(depth, child_bays) sub_row = self._build_row(sub_item, index_map, scope_bays, module_types, depth=depth) @@ -228,7 +228,7 @@ def _build_context(self, request, obj, inventory_data): # Mark parent if any child is installable if sub_row.get("can_install"): - table_data[parent_idx]["has_installable_children"] = True + table_data[parent_row_idx]["has_installable_children"] = True # When parent is installable but children can't match bays yet # (parent module not installed), enable "Install Branch" if any child @@ -236,7 +236,7 @@ def _build_context(self, request, obj, inventory_data): if ( parent_bay_matched_but_uninstalled and row.get("can_install") - and not table_data[parent_idx].get("has_installable_children") + and not table_data[parent_row_idx].get("has_installable_children") ): for _depth, sub_item in sub_items: sub_model = (sub_item.get("entPhysicalModelName") or "").strip() @@ -251,7 +251,7 @@ def _build_context(self, request, obj, inventory_data): ) matched = module_types.get(normalized) if matched: - table_data[parent_idx]["has_installable_children"] = True + table_data[parent_row_idx]["has_installable_children"] = True break # Sort top-level groups by status, keeping children after their parent @@ -286,7 +286,7 @@ def _merge_transceiver_data(self, inventory_data): return inventory_data # Build lookup of existing inventory items by index and serial - inv_by_index = {item["entPhysicalIndex"]: item for item in inventory_data} + inv_by_index = {idx: item for item in inventory_data if (idx := item.get("entPhysicalIndex")) is not None} inv_serials = { (item.get("entPhysicalSerialNum") or "").strip() for item in inventory_data @@ -386,7 +386,9 @@ def _collect_descendants(self, parent_idx, inventory_data, depth, results, visit visited = set() children = [i for i in inventory_data if i.get("entPhysicalContainedIn") == parent_idx] for child in children: - child_idx = child["entPhysicalIndex"] + child_idx = child.get("entPhysicalIndex") + if child_idx is None: + continue if child_idx in visited: continue visited.add(child_idx) diff --git a/netbox_librenms_plugin/views/sync/modules.py b/netbox_librenms_plugin/views/sync/modules.py index 2f90154b83..dc44e7857e 100644 --- a/netbox_librenms_plugin/views/sync/modules.py +++ b/netbox_librenms_plugin/views/sync/modules.py @@ -110,6 +110,13 @@ def post(self, request, pk): # Load module types (with mappings) module_types = self._get_module_types() + # Preload all ModuleBayMappings once to avoid N+1 per-item queries + from netbox_librenms_plugin.models import ModuleBayMapping + + all_mappings = list(ModuleBayMapping.objects.all()) + exact_mappings = [m for m in all_mappings if not m.is_regex] + regex_mappings = [m for m in all_mappings if m.is_regex] + # Install top-down: each install may create new child bays installed = [] skipped = [] @@ -126,6 +133,8 @@ def post(self, request, pk): ModuleBay, ModuleType, Module, + exact_mappings=exact_mappings, + regex_mappings=regex_mappings, ) if result["status"] == "installed": installed.append(result["name"]) @@ -156,7 +165,7 @@ def _collect_branch(self, parent_index, inventory_data): Returns items in install order (parent before children). """ items = [] - parent = next((i for i in inventory_data if i["entPhysicalIndex"] == parent_index), None) + parent = next((i for i in inventory_data if i.get("entPhysicalIndex") == parent_index), None) if parent: model = (parent.get("entPhysicalModelName") or "").strip() if model: @@ -170,7 +179,9 @@ def _collect_children(self, parent_idx, inventory_data, items, visited=None): visited = set() children = [i for i in inventory_data if i.get("entPhysicalContainedIn") == parent_idx] for child in children: - child_idx = child["entPhysicalIndex"] + child_idx = child.get("entPhysicalIndex") + if child_idx is None: + continue if child_idx in visited: continue visited.add(child_idx) @@ -196,13 +207,23 @@ def _get_module_types(self): result[mapping.librenms_model] = mapping.netbox_module_type return result - def _install_single(self, device, item, index_map, module_types, ModuleBay, ModuleType, Module): + def _install_single( + self, + device, + item, + index_map, + module_types, + ModuleBay, + ModuleType, + Module, + exact_mappings=None, + regex_mappings=None, + ): """Try to install a single inventory item. Re-fetches module bays each time since parent installs create new ones. Scopes bay lookup to the correct parent module to handle duplicate bay names. """ - from netbox_librenms_plugin.models import ModuleBayMapping from netbox_librenms_plugin.utils import apply_normalization_rules model_name = (item.get("entPhysicalModelName") or "").strip() @@ -222,8 +243,15 @@ def _install_single(self, device, item, index_map, module_types, ModuleBay, Modu # Re-fetch module bays (parent install creates new child bays) bays = ModuleBay.objects.filter(device=device).select_related("installed_module__module_type") - # Pre-fetch all bay mappings once to avoid N+1 queries in _find_parent_module_id - bay_mappings = list(ModuleBayMapping.objects.all()) + # Use preloaded mappings if provided, otherwise load from DB + if exact_mappings is None or regex_mappings is None: + from netbox_librenms_plugin.models import ModuleBayMapping + + all_mappings = list(ModuleBayMapping.objects.all()) + exact_mappings = [m for m in all_mappings if not m.is_regex] + regex_mappings = [m for m in all_mappings if m.is_regex] + + bay_mappings = exact_mappings + regex_mappings # Determine if this item belongs under an installed module # by tracing its LibreNMS parent hierarchy to an installed item @@ -234,8 +262,8 @@ def _install_single(self, device, item, index_map, module_types, ModuleBay, Modu else: bay_dict = {bay.name: bay for bay in bays if not bay.module_id} - # Match module bay using mapping table - matched_bay = self._match_bay(item, index_map, bay_dict, ModuleBayMapping) + # Match module bay using preloaded mapping data + matched_bay = self._match_bay(item, index_map, bay_dict, exact_mappings, regex_mappings) if not matched_bay: return {"status": "skipped", "name": name, "reason": "no matching bay"} @@ -293,10 +321,14 @@ def _find_parent_module_id(item, index_map, device_bays, bay_mappings): if m.librenms_name not in mapping_by_name: mapping_by_name[m.librenms_name] = m - for _ in range(10): # max depth guard + visited = set() + while True: parent_idx = current.get("entPhysicalContainedIn", 0) if not parent_idx or parent_idx not in index_map: return None + if parent_idx in visited: + return None + visited.add(parent_idx) parent = index_map[parent_idx] parent_name = parent.get("entPhysicalName", "") parent_descr = parent.get("entPhysicalDescr", "") @@ -318,22 +350,30 @@ def _find_parent_module_id(item, index_map, device_bays, bay_mappings): return bay.installed_module.pk current = parent - return None @staticmethod - def _match_bay(item, index_map, module_bays, ModuleBayMapping): + def _match_bay(item, index_map, module_bays, exact_mappings, regex_mappings): """Match an inventory item to a module bay (same logic as BaseModuleTableView).""" import re from netbox_librenms_plugin.views.base.modules_view import BaseModuleTableView - # Resolve parent name + # Resolve parent name by walking up the containment hierarchy contained_in = item.get("entPhysicalContainedIn", 0) parent_name = None if contained_in: - parent = index_map.get(contained_in) - if parent: - parent_name = parent.get("entPhysicalName", "") + visited_anc = set() + current_idx = contained_in + while current_idx and current_idx not in visited_anc: + visited_anc.add(current_idx) + ancestor = index_map.get(current_idx) + if not ancestor: + break + ancestor_name = ancestor.get("entPhysicalName", "") + if ancestor_name: + parent_name = ancestor_name + break + current_idx = ancestor.get("entPhysicalContainedIn", 0) item_name = item.get("entPhysicalName", "") item_descr = item.get("entPhysicalDescr", "") @@ -342,20 +382,23 @@ def _match_bay(item, index_map, module_bays, ModuleBayMapping): # Build candidate names: parent, item name, item description candidate_names = [n for n in [parent_name, item_name, item_descr] if n] - # Check mapping for each candidate (exact match) + # Check mapping for each candidate (exact match, in-memory lookup) + # Group exact_mappings by (librenms_name, librenms_class) for O(1) lookup + exact_by_name: dict = {} + for m in exact_mappings: + exact_by_name.setdefault(m.librenms_name, []).append(m) + for name in candidate_names: - filters = {"librenms_name": name, "is_regex": False} + candidates_for_name = exact_by_name.get(name, []) + mapping = None if phys_class: - mapping = ModuleBayMapping.objects.filter(**filters, librenms_class=phys_class).first() - if not mapping: - mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() - else: - mapping = ModuleBayMapping.objects.filter(**filters, librenms_class="").first() + mapping = next((m for m in candidates_for_name if m.librenms_class == phys_class), None) + if not mapping: + mapping = next((m for m in candidates_for_name if m.librenms_class == ""), None) if mapping and mapping.netbox_bay_name in module_bays: return module_bays[mapping.netbox_bay_name] - # Regex pattern matching on all candidate names (preload once to avoid N+1) - regex_mappings = list(ModuleBayMapping.objects.filter(is_regex=True)) + # Regex pattern matching using preloaded list for name in candidate_names: bay = BaseModuleTableView._lookup_regex_bay_mapping(re, name, phys_class, module_bays, regex_mappings) if bay: @@ -419,13 +462,28 @@ def post(self, request, pk): helper = InstallBranchView() module_types = helper._get_module_types() + # Preload all ModuleBayMappings once to avoid N+1 per-item queries + from netbox_librenms_plugin.models import ModuleBayMapping + + all_mappings = list(ModuleBayMapping.objects.all()) + exact_mappings = [m for m in all_mappings if not m.is_regex] + regex_mappings = [m for m in all_mappings if m.is_regex] + installed, skipped, failed = [], [], [] try: with transaction.atomic(): for item in items: result = helper._install_single( - device, item, index_map, module_types, ModuleBay, ModuleType, Module + device, + item, + index_map, + module_types, + ModuleBay, + ModuleType, + Module, + exact_mappings=exact_mappings, + regex_mappings=regex_mappings, ) if result["status"] == "installed": installed.append(result["name"]) From dd479109990e88acc00013eeab5ff0899051f414 Mon Sep 17 00:00:00 2001 From: Marcin Zieba Date: Thu, 5 Mar 2026 19:04:26 +0100 Subject: [PATCH 27/28] fix(modules): inventory cache server-namespace, O(n) descendants, vm comments device_id, tests - CacheMixin.get_cache_key() accepts optional server_key param for per-server namespacing - modules_view: inventory cache set/get/ttl use server_key from librenms_api - tables/modules: LibreNMSModuleTable stores server_key, passes it in inline install forms - sync/modules: install views read server_key from POST, use it for cache.get/delete - _module_sync_content.html: add server_key hidden input to install-selected-form - object_sync/devices: DeviceModuleTableView.get_table passes server_key to table - modules_view: _collect_descendants refactored to O(n) via precomputed children_by_parent - modules_view: synthetic transceiver deduplication: update inv_by_index/inv_serials after append - modules_view: match.expand() wrapped in try/except re.error to prevent runtime crash - vm_operations: comments string includes device_id for traceability - test_modules_view: _make_view sets _librenms_api mock; _collect_descendants tests use new signature - test_sync_modules: regression test for parent_row_idx vs entity index collision - test_vm_operations: statuses iterator fix; device_id assertion in comments --- .../import_utils/vm_operations.py | 2 +- netbox_librenms_plugin/tables/modules.py | 7 +- .../_module_sync_content.html | 1 + .../tests/test_modules_view.py | 15 +++- .../tests/test_sync_modules.py | 81 +++++++++++++++++++ .../tests/test_tables_modules.py | 1 + .../tests/test_vm_operations.py | 3 +- .../views/base/modules_view.py | 36 ++++++--- netbox_librenms_plugin/views/mixins.py | 10 ++- .../views/object_sync/devices.py | 2 +- netbox_librenms_plugin/views/sync/modules.py | 14 ++-- 11 files changed, 147 insertions(+), 25 deletions(-) diff --git a/netbox_librenms_plugin/import_utils/vm_operations.py b/netbox_librenms_plugin/import_utils/vm_operations.py index 4017cc97f7..2ca9af2a97 100644 --- a/netbox_librenms_plugin/import_utils/vm_operations.py +++ b/netbox_librenms_plugin/import_utils/vm_operations.py @@ -69,7 +69,7 @@ def create_vm_from_librenms( cluster=cluster, role=role, # Optional VM role platform=platform, - comments=f"Imported from LibreNMS by netbox-librenms-plugin on {import_time}", + comments=f"Imported from LibreNMS (device_id={librenms_device_id}) by netbox-librenms-plugin on {import_time}", ) set_librenms_device_id(vm, librenms_device_id, server_key) vm.save() diff --git a/netbox_librenms_plugin/tables/modules.py b/netbox_librenms_plugin/tables/modules.py index 87788eb19a..4ab417cdad 100644 --- a/netbox_librenms_plugin/tables/modules.py +++ b/netbox_librenms_plugin/tables/modules.py @@ -40,10 +40,11 @@ class Meta: attrs = {"class": "table table-hover object-list", "id": "librenms-module-table"} row_attrs = {"class": lambda record: record.get("row_class", "")} - def __init__(self, *args, device=None, **kwargs): + def __init__(self, *args, device=None, server_key="", **kwargs): """Initialize table with optional device context.""" self.device = device self.csrf_token = "" + self.server_key = server_key super().__init__(*args, **kwargs) self.tab = "modules" self.htmx_url = None @@ -173,6 +174,7 @@ def render_actions(self, value, record): format_html( '
' '' + '' '' '' '' @@ -181,6 +183,7 @@ def render_actions(self, value, record): "
", url, self.csrf_token, + self.server_key, record.get("module_bay_id", ""), record.get("module_type_id", ""), record.get("serial", ""), @@ -194,6 +197,7 @@ def render_actions(self, value, record): format_html( '
' '' + '' '' '
", url, self.csrf_token, + self.server_key, record.get("ent_physical_index", ""), ) ) diff --git a/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html b/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html index 4424c53f43..afd1dd5c28 100644 --- a/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html +++ b/netbox_librenms_plugin/templates/netbox_librenms_plugin/_module_sync_content.html @@ -20,6 +20,7 @@
{% csrf_token %} +